Bug 1808605 - part 23: Add browsertime and startup-test kinds
(cherry picked from commit f536f895bb13056b645ed90c90bdf6f279d57a9d)pull/600/head
Geoff Brown
2 years ago
committed by
mergify[bot]
parent
543c941969
commit
ebe2934ba9
@ -1,158 +0,0 @@
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
transforms:
|
||||
- fenix_taskgraph.transforms.browsertime:transforms
|
||||
- fenix_taskgraph.transforms.notify:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- signing
|
||||
- toolchain
|
||||
|
||||
primary-dependency: signing
|
||||
|
||||
only-for-build-types:
|
||||
- nightly-simulation
|
||||
|
||||
only-for-abis:
|
||||
- armeabi-v7a
|
||||
|
||||
task-defaults:
|
||||
attributes:
|
||||
artifact_prefix: public/test_info
|
||||
nightly-test: true
|
||||
dependencies:
|
||||
geckoview-nightly: geckoview-nightly
|
||||
notify:
|
||||
by-level:
|
||||
'3':
|
||||
email:
|
||||
content: This calls for an action of the Performance team. Use the link to view it on Treeherder.
|
||||
link:
|
||||
text: Treeherder Job
|
||||
href: 'https://treeherder.mozilla.org/#/jobs?repo={product_name}&revision={head_rev}&searchStr={task_name}'
|
||||
on-reasons: [failed]
|
||||
subject: '[{product_name}] Raptor-Browsertime job "{task_name}" failed'
|
||||
to-addresses: [perftest-alerts@mozilla.com]
|
||||
default: {}
|
||||
run-on-tasks-for: []
|
||||
treeherder:
|
||||
kind: test
|
||||
tier: 2
|
||||
platform:
|
||||
by-abi:
|
||||
armeabi-v7a: android-hw-a51-11-0-aarch64-shippable-qr/opt
|
||||
worker-type:
|
||||
by-abi:
|
||||
armeabi-v7a: t-bitbar-gw-perf-a51
|
||||
worker:
|
||||
max-run-time: 3600
|
||||
env:
|
||||
GECKO_HEAD_REPOSITORY: "https://hg.mozilla.org/mozilla-central"
|
||||
MOZ_AUTOMATION: "1"
|
||||
MOZ_HIDE_RESULTS_TABLE: "1"
|
||||
MOZ_NO_REMOTE: "1"
|
||||
MOZ_NODE_PATH: "/usr/local/bin/node"
|
||||
MOZHARNESS_CONFIG: "raptor/android_hw_config.py"
|
||||
MOZHARNESS_SCRIPT: "raptor_script.py"
|
||||
NEED_XVFB: "false"
|
||||
NO_FAIL_ON_TEST_ERRORS: "1"
|
||||
XPCOM_DEBUG_BREAK: "warn"
|
||||
PYTHON: "python3"
|
||||
LANG: "en_US.UTF-8"
|
||||
artifacts:
|
||||
- name: public/logs/
|
||||
path: workspace/logs
|
||||
type: directory
|
||||
|
||||
- name: public/test_info/
|
||||
path: workspace/build/blobber_upload_dir
|
||||
type: directory
|
||||
run-visual-metrics: True
|
||||
chimera: True
|
||||
run:
|
||||
using: run-task
|
||||
checkout: false
|
||||
run-as-root: true
|
||||
command:
|
||||
- 'bash'
|
||||
- './test-linux.sh'
|
||||
- '--cfg=mozharness/configs/raptor/android_hw_config.py'
|
||||
- '--app=fenix'
|
||||
- '--browsertime'
|
||||
- '--binary=org.mozilla.fenix'
|
||||
- '--activity=org.mozilla.fenix.IntentReceiverActivity'
|
||||
- '--download-symbols=ondemand'
|
||||
- '--browsertime-node=$MOZ_FETCHES_DIR/node/bin/node'
|
||||
- '--browsertime-geckodriver=$MOZ_FETCHES_DIR/geckodriver'
|
||||
- '--browsertime-ffmpeg=$MOZ_FETCHES_DIR/ffmpeg-4.1.4-i686-static/bin/ffmpeg'
|
||||
- '--browsertime-browsertimejs=$MOZ_FETCHES_DIR/browsertime/node_modules/browsertime/bin/browsertime.js'
|
||||
- '--conditioned-profile=settled'
|
||||
fetches:
|
||||
toolchain:
|
||||
- browsertime
|
||||
- linux64-ffmpeg-4.1.4
|
||||
- linux64-geckodriver
|
||||
- linux64-minidump-stackwalk
|
||||
- linux64-node-16
|
||||
|
||||
tasks:
|
||||
tp6m:
|
||||
run-with-fission: True
|
||||
page-load-tests:
|
||||
- amazon
|
||||
- instagram
|
||||
- [bing-search-restaurants, bing-s-r]
|
||||
- [ebay-kleinanzeigen-search, ebay-k-s]
|
||||
- wikipedia
|
||||
- booking
|
||||
- [cnn-ampstories, cnn-amp]
|
||||
- dailymail
|
||||
- imdb
|
||||
- [facebook-cristiano, fb-cris]
|
||||
- youtube
|
||||
- bing
|
||||
- [ebay-kleinanzeigen, ebay-k]
|
||||
- [google-maps, gmaps]
|
||||
- reddit
|
||||
- sina
|
||||
- [stackoverflow, stacko]
|
||||
- web-de
|
||||
- cnn
|
||||
- [google-search-restaurants, gsearch-r]
|
||||
|
||||
tp6m-hv:
|
||||
run-with-fission: True
|
||||
page-load-tests:
|
||||
- google
|
||||
- [amazon-search, amazon-s]
|
||||
- [microsoft-support, micros-sup]
|
||||
- espn
|
||||
- facebook
|
||||
- [youtube-watch, youtube-w]
|
||||
- allrecipes
|
||||
|
||||
speedometer3:
|
||||
description: "Raptor Speedometer 3 Benchmark tests on Fenix"
|
||||
test-name: speedometer3
|
||||
run-visual-metrics: False
|
||||
chimera: False
|
||||
|
||||
youtube-playback-av1-sfr:
|
||||
description: "Raptor YouTube Playback AV1 SFR on Fenix"
|
||||
test-name: youtube-playback-av1-sfr
|
||||
run-visual-metrics: False
|
||||
chimera: False
|
||||
|
||||
youtube-playback-h264-sfr:
|
||||
description: "Raptor YouTube Playback H264 SFR on Fenix"
|
||||
test-name: youtube-playback-h264-sfr
|
||||
run-visual-metrics: False
|
||||
chimera: False
|
||||
|
||||
youtube-playback-vp9-sfr:
|
||||
description: "Raptor YouTube Playback VP9 SFR on Fenix"
|
||||
run-visual-metrics: False
|
||||
chimera: False
|
||||
test-name: youtube-playback-vp9-sfr
|
||||
@ -1,17 +0,0 @@
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
transforms:
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
task-defaults:
|
||||
worker-type: always-optimized
|
||||
run:
|
||||
using: index-search
|
||||
|
||||
tasks:
|
||||
nightly:
|
||||
description: "upstream nightly job"
|
||||
run:
|
||||
index-search:
|
||||
- gecko.v2.mozilla-central.shippable.latest.mobile.android-x86_64-opt
|
||||
@ -1,55 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
task-defaults:
|
||||
description: Run a simple UI test to verify Nightly build can start
|
||||
treeherder:
|
||||
kind: test
|
||||
platform: 'nightly-start-test/opt'
|
||||
tier: 2
|
||||
worker-type: b-android
|
||||
worker:
|
||||
docker-image: {in-tree: ui-tests}
|
||||
max-run-time: 7200
|
||||
env:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json'
|
||||
GOOGLE_PROJECT: moz-fenix
|
||||
artifacts:
|
||||
- name: public
|
||||
path: /builds/worker/artifacts
|
||||
type: directory
|
||||
run:
|
||||
using: run-commands
|
||||
use-caches: false
|
||||
secrets:
|
||||
- name: project/mobile/fenix/firebase
|
||||
key: firebaseToken
|
||||
path: .firebase_token.json
|
||||
json: true
|
||||
run-on-tasks-for: []
|
||||
attributes:
|
||||
build-type: nightly
|
||||
# cannot safely run this in Firebase
|
||||
# turning off til we can safely run test in taskcluster
|
||||
# https://github.com/mozilla-mobile/fenix/issues/7702
|
||||
#nightly: true
|
||||
dependencies:
|
||||
signing: signing-nightly
|
||||
signing-android-test: signing-android-test-nightly
|
||||
|
||||
tasks:
|
||||
nightly-arm:
|
||||
run:
|
||||
commands:
|
||||
- [wget, {artifact-reference: '<signing/public/build/arm64-v8a/target.apk>'}, '-O', app.apk]
|
||||
- [wget, {artifact-reference: '<signing-android-test/public/build/noarch/target.apk>'}, '-O', android-test.apk]
|
||||
- [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app.apk, android-test.apk, '1']
|
||||
treeherder:
|
||||
symbol: nightly(startup-arm)
|
||||
@ -1,189 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
from copy import deepcopy
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def split_raptor_subtests(config, tests):
|
||||
for test in tests:
|
||||
# For tests that have 'page-load-tests' listed, we want to create a separate
|
||||
# test job for every subtest (i.e. split out each page-load URL into its own job)
|
||||
subtests = test.pop("page-load-tests", None)
|
||||
if not subtests:
|
||||
yield test
|
||||
continue
|
||||
|
||||
for subtest in subtests:
|
||||
pageload_test = deepcopy(test)
|
||||
|
||||
if isinstance(subtest, list):
|
||||
pageload_test["test-name"] = subtest[0]
|
||||
pageload_test["subtest-symbol"] = subtest[1]
|
||||
else:
|
||||
pageload_test["test-name"] = subtest
|
||||
pageload_test["subtest-symbol"] = subtest
|
||||
yield pageload_test
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_variants(config, tasks):
|
||||
only_types = config.config["only-for-build-types"]
|
||||
only_abis = config.config["only-for-abis"]
|
||||
|
||||
tests = list(tasks)
|
||||
|
||||
for dep_task in config.kind_dependencies_tasks.values():
|
||||
build_type = dep_task.attributes.get("build-type", "")
|
||||
if build_type not in only_types:
|
||||
continue
|
||||
|
||||
for abi, apk_metadata in dep_task.attributes["apks"].items():
|
||||
if abi not in only_abis:
|
||||
continue
|
||||
apk_path = apk_metadata["name"]
|
||||
for test in tests:
|
||||
test = copy.deepcopy(test)
|
||||
attributes = copy.deepcopy(dep_task.attributes)
|
||||
attributes.update(test.get("attributes", {}))
|
||||
attributes["abi"] = abi
|
||||
attributes["apk"] = apk_path
|
||||
test["attributes"] = attributes
|
||||
test["primary-dependency"] = dep_task
|
||||
yield test
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_browsertime_task(config, tasks):
|
||||
for task in tasks:
|
||||
signing = task.pop("primary-dependency")
|
||||
task.setdefault("dependencies", {})["signing"] = signing.label
|
||||
build_type = task["attributes"]["build-type"]
|
||||
abi = task["attributes"]["abi"]
|
||||
apk = task["attributes"]["apk"]
|
||||
|
||||
for key in ("args", "treeherder.platform", "worker-type"):
|
||||
resolve_keyed_by(task, key, item_name=task["name"], **{"abi": abi})
|
||||
|
||||
task["treeherder"] = inherit_treeherder_from_dep(task, signing)
|
||||
|
||||
test_name = task.pop("test-name")
|
||||
platform = task["treeherder"]["platform"]
|
||||
|
||||
task_name = "{}-{}-{}-{}".format(platform, test_name, build_type, abi)
|
||||
task["name"] = task_name
|
||||
task["description"] = task_name
|
||||
|
||||
extra_config = {
|
||||
"installer_url": "<signing/{}>".format(apk),
|
||||
"test_packages_url": "<geckoview-nightly/public/build/en-US/target.test_packages.json>",
|
||||
}
|
||||
env = task["worker"]["env"]
|
||||
env["EXTRA_MOZHARNESS_CONFIG"] = {
|
||||
"artifact-reference": json.dumps(extra_config, sort_keys=True)
|
||||
}
|
||||
env["GECKO_HEAD_REV"] = "default"
|
||||
env["MOZILLA_BUILD_URL"] = {"artifact-reference": "<signing/{}>".format(apk)}
|
||||
env["MOZHARNESS_URL"] = {
|
||||
"artifact-reference": "<geckoview-nightly/public/build/en-US/mozharness.zip>"
|
||||
}
|
||||
env["TASKCLUSTER_WORKER_TYPE"] = task["worker-type"]
|
||||
|
||||
worker = task["worker"]
|
||||
worker.setdefault("mounts", []).append(
|
||||
{
|
||||
"content": {
|
||||
"url": "https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/scripts/tester/test-linux.sh"
|
||||
},
|
||||
"file": "./test-linux.sh",
|
||||
}
|
||||
)
|
||||
task["run"]["command"].append("--test={}".format(test_name))
|
||||
task["run"]["command"].extend(task.pop("args", []))
|
||||
|
||||
# Setup treherder symbol
|
||||
symbol = task.pop("subtest-symbol", None)
|
||||
|
||||
# taskcluster is merging task attributes with the default ones
|
||||
# resulting the --cold extra option in the ytp warm tasks
|
||||
if "youtube-playback" in task["name"]:
|
||||
symbol = test_name.replace("youtube-playback-", "ytp-")
|
||||
|
||||
# Setup chimera for combined warm+cold testing
|
||||
if task.pop("chimera", False):
|
||||
task["run"]["command"].append("--chimera")
|
||||
|
||||
# Add '-c' to taskcluster symbol when running cold tests
|
||||
elif "--cold" in task["run"]["command"]:
|
||||
symbol += "-c"
|
||||
|
||||
# Setup visual metrics
|
||||
run_visual_metrics = task.pop("run-visual-metrics", False)
|
||||
if run_visual_metrics:
|
||||
task["run"]["command"].append("--browsertime-video")
|
||||
task["run"]["command"].append("--browsertime-visualmetrics")
|
||||
task["run"]["command"].append("--browsertime-no-ffwindowrecorder")
|
||||
|
||||
# Build taskcluster group and symol
|
||||
task["treeherder"]["symbol"] = "Btime(%s)" % symbol
|
||||
task["name"] = (
|
||||
task["name"].replace("tp6m-", "tp6m-{}-".format(symbol)).replace("-hv", "")
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def setup_nofis(config, tasks):
|
||||
for task in tasks:
|
||||
if task.pop("run-with-fission", False):
|
||||
# Don't continue after this since this flag says
|
||||
# that a fission and a non-fission variant should
|
||||
# run
|
||||
fission_task = copy.deepcopy(task)
|
||||
yield fission_task
|
||||
|
||||
# Disable fission
|
||||
task["run"]["command"].append("--disable-fission")
|
||||
|
||||
# Build taskcluster group and symol
|
||||
task["treeherder"]["symbol"] = (
|
||||
task["treeherder"]["symbol"].replace("Btime", "Btime-nofis")
|
||||
)
|
||||
task["name"] += "-nofis"
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def fill_email_data(config, tasks):
|
||||
product_name = config.graph_config["taskgraph"]["repositories"]["mobile"]["name"]
|
||||
format_kwargs = {
|
||||
"product_name": product_name.lower(),
|
||||
"head_rev": config.params["head_rev"],
|
||||
}
|
||||
|
||||
for task in tasks:
|
||||
format_kwargs["task_name"] = task["name"]
|
||||
|
||||
resolve_keyed_by(
|
||||
task, "notify", item_name=task["name"], level=config.params["level"]
|
||||
)
|
||||
email = task["notify"].get("email")
|
||||
if email:
|
||||
email["link"]["href"] = email["link"]["href"].format(**format_kwargs)
|
||||
email["subject"] = email["subject"].format(**format_kwargs)
|
||||
|
||||
yield task
|
||||
Loading…
Reference in New Issue