diff --git a/taskcluster/ci/browsertime/kind.yml b/taskcluster/ci/browsertime/kind.yml index ea014660d2..2c2f7d6a15 100644 --- a/taskcluster/ci/browsertime/kind.yml +++ b/taskcluster/ci/browsertime/kind.yml @@ -83,7 +83,6 @@ job-defaults: - '--cfg=mozharness/configs/raptor/android_hw_config.py' - '--app=fenix' - '--browsertime' - - '--cold' - '--no-conditioned-profile' - '--binary=org.mozilla.fenix' - '--activity=org.mozilla.fenix.IntentReceiverActivity' @@ -101,168 +100,54 @@ job-defaults: - linux64-node jobs: - tp6m-1-cold: - test-name: amazon - treeherder: - symbol: 'Btime(tp6m-1-c)' + tp6m: + page-load-tests: + - amazon + - instagram + - [bing-search-restaurants, bing-s-r] + - [ebay-kleinanzeigen-search, ebay-k-s] + - wikipedia + - booking + - [cnn-ampstories, cnn-amp] + - bbc + - imdb + - [facebook-cristiano, fb-cris] + - youtube + - bing + - [ebay-kleinanzeigen, ebay-k] + - [google-maps, gmaps] + - reddit + - [stackoverflow, stacko] + - jianshu + - web-de + - cnn + - [google-search-restaurants, gsearch-r] - tp6m-2-cold: - test-name: google + tp6m-hv: web-render-only: False - treeherder: - symbol: 'Btime(tp6m-2-c)' - - tp6m-3-cold: - test-name: instagram - treeherder: - symbol: 'Btime(tp6m-3-c)' - - tp6m-4-cold: - test-name: bing-search-restaurants - treeherder: - symbol: 'Btime(tp6m-4-c)' - - tp6m-5-cold: - test-name: ebay-kleinanzeigen-search - treeherder: - symbol: 'Btime(tp6m-5-c)' - - tp6m-6-cold: - test-name: amazon-search - web-render-only: False - treeherder: - symbol: 'Btime(tp6m-6-c)' - - tp6m-7-cold: - test-name: wikipedia - treeherder: - symbol: 'Btime(tp6m-7-c)' - - tp6m-8-cold: - test-name: booking - treeherder: - symbol: 'Btime(tp6m-8-c)' - - tp6m-9-cold: - test-name: cnn-ampstories - treeherder: - symbol: 'Btime(tp6m-9-c)' - - tp6m-10-cold: - test-name: bbc - treeherder: - symbol: 'Btime(tp6m-10-c)' - - tp6m-11-cold: - test-name: microsoft-support - web-render-only: False - treeherder: - symbol: 'Btime(tp6m-11-c)' - - tp6m-12-cold: - test-name: imdb - treeherder: - symbol: 'Btime(tp6m-12-c)' - - tp6m-13-cold: - test-name: espn - web-render-only: False - treeherder: - symbol: 'Btime(tp6m-13-c)' - - tp6m-14-cold: - test-name: facebook-cristiano - treeherder: - symbol: 'Btime(tp6m-14-c)' - - tp6m-15-cold: - test-name: facebook - web-render-only: False - treeherder: - symbol: 'Btime(tp6m-15-c)' - - tp6m-16-cold: - test-name: youtube - treeherder: - symbol: 'Btime(tp6m-16-c)' - - tp6m-17-cold: - test-name: bing - treeherder: - symbol: 'Btime(tp6m-17-c)' - - tp6m-18-cold: - test-name: ebay-kleinanzeigen - treeherder: - symbol: 'Btime(tp6m-18-c)' - - tp6m-19-cold: - test-name: google-maps - treeherder: - symbol: 'Btime(tp6m-19-c)' - - tp6m-20-cold: - test-name: youtube-watch - web-render-only: False - treeherder: - symbol: 'Btime(tp6m-20-c)' - - tp6m-21-cold: - test-name: reddit - treeherder: - symbol: 'Btime(tp6m-21-c)' - - tp6m-22-cold: - test-name: stackoverflow - treeherder: - symbol: 'Btime(tp6m-22-c)' - - tp6m-23-cold: - test-name: jianshu - treeherder: - symbol: 'Btime(tp6m-23-c)' - - tp6m-24-cold: - test-name: allrecipes - web-render-only: False - treeherder: - symbol: 'Btime(tp6m-24-c)' - - tp6m-25-cold: - test-name: web-de - treeherder: - symbol: 'Btime(tp6m-25-c)' - - tp6m-27-cold: - test-name: cnn - treeherder: - symbol: 'Btime(tp6m-27-c)' - - tp6m-28-cold: - test-name: google-search-restaurants - treeherder: - symbol: 'Btime(tp6m-28-c)' + page-load-tests: + - google + - [amazon-search, amazon-s] + - [microsoft-support, micros-sup] + - espn + - facebook + - [youtube-watch, youtube-w] + - allrecipes youtube-playback-av1-sfr: description: "Raptor YouTube Playback AV1 SFR on Fenix" test-name: youtube-playback-av1-sfr run-visual-metrics: False chimera: False - treeherder: - symbol: 'Btime(ytp-av1-sfr)' youtube-playback-h264-sfr: description: "Raptor YouTube Playback H264 SFR on Fenix" test-name: youtube-playback-h264-sfr run-visual-metrics: False chimera: False - treeherder: - symbol: Btime(ytp-h264-sfr) youtube-playback-vp9-sfr: description: "Raptor YouTube Playback VP9 SFR on Fenix" run-visual-metrics: False chimera: False test-name: youtube-playback-vp9-sfr - treeherder: - symbol: Btime(ytp-vp9-sfr) diff --git a/taskcluster/fenix_taskgraph/transforms/browsertime.py b/taskcluster/fenix_taskgraph/transforms/browsertime.py index 9f5e915723..544d068609 100644 --- a/taskcluster/fenix_taskgraph/transforms/browsertime.py +++ b/taskcluster/fenix_taskgraph/transforms/browsertime.py @@ -11,6 +11,7 @@ from __future__ import absolute_import, print_function, unicode_literals import copy import json +from copy import deepcopy from taskgraph.transforms.base import TransformSequence from taskgraph.util.treeherder import inherit_treeherder_from_dep from taskgraph.util.schema import resolve_keyed_by @@ -18,6 +19,28 @@ from taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() +@transforms.add +def split_raptor_subtests(config, tests): + for test in tests: + # For tests that have 'page-load-tests' listed, we want to create a separate + # test job for every subtest (i.e. split out each page-load URL into its own job) + subtests = test.pop("page-load-tests", None) + if not subtests: + yield test + continue + + for subtest in subtests: + pageload_test = deepcopy(test) + + if isinstance(subtest, list): + pageload_test["test-name"] = subtest[0] + pageload_test["subtest-symbol"] = subtest[1] + else: + pageload_test["test-name"] = subtest + pageload_test["subtest-symbol"] = subtest + yield pageload_test + + @transforms.add def add_variants(config, tasks): only_types = config.config["only-for-build-types"] @@ -26,7 +49,7 @@ def add_variants(config, tasks): tests = list(tasks) for dep_task in config.kind_dependencies_tasks: - build_type = dep_task.attributes.get("build-type", '') + build_type = dep_task.attributes.get("build-type", "") if build_type not in only_types: continue @@ -91,6 +114,22 @@ def build_browsertime_task(config, tasks): task["run"]["command"].append("--test={}".format(test_name)) task["run"]["command"].extend(task.pop("args", [])) + # Setup treherder symbol + symbol = task.pop("subtest-symbol", None) + + # taskcluster is merging task attributes with the default ones + # resulting the --cold extra option in the ytp warm tasks + if "youtube-playback" in task["name"]: + symbol = test_name.replace("youtube-playback-", "ytp-") + + # Setup chimera for combined warm+cold testing + if task.pop("chimera", False): + task["run"]["command"].append("--chimera") + + # Add '-c' to taskcluster symbol when running cold tests + elif "--cold" in task["run"]["command"]: + symbol += "-c" + # Setup visual metrics run_visual_metrics = task.pop("run-visual-metrics", False) if run_visual_metrics: @@ -98,15 +137,11 @@ def build_browsertime_task(config, tasks): task["run"]["command"].append("--browsertime-no-ffwindowrecorder") task["attributes"]["run-visual-metrics"] = True - # Setup chimera for combined warm+cold testing - if task.pop("chimera", False): - task["run"]["command"].append("--chimera") - - # taskcluster is merging task attributes with the default ones - # resulting the --cold extra option in the ytp warm tasks - if 'youtube-playback' in task["name"]: - task["run"]["command"].remove("--cold") - + # Build taskcluster group and symol + task["treeherder"]["symbol"] = "Btime(%s)" % symbol + task["name"] = ( + task["name"].replace("tp6m-", "tp6m-{}-".format(symbol)).replace("-hv", "") + ) yield task @@ -129,7 +164,7 @@ def enable_webrender(config, tasks): @transforms.add def fill_email_data(config, tasks): - product_name = config.graph_config['taskgraph']['repositories']['mobile']['name'] + product_name = config.graph_config["taskgraph"]["repositories"]["mobile"]["name"] format_kwargs = { "product_name": product_name.lower(), "head_rev": config.params["head_rev"], @@ -138,7 +173,9 @@ def fill_email_data(config, tasks): for task in tasks: format_kwargs["task_name"] = task["name"] - resolve_keyed_by(task, 'notify', item_name=task["name"], level=config.params["level"]) + resolve_keyed_by( + task, "notify", item_name=task["name"], level=config.params["level"] + ) email = task["notify"].get("email") if email: email["link"]["href"] = email["link"]["href"].format(**format_kwargs)