* For #26792 - Change active Firebase Test Lab emulator from x86 to ARM Update startup-test kind.yml Jobs for ARM Update ui-test kind.yml Jobs for ARM Cleans up and renames Flank configurations for ARM Updates ui-test.sh for ARM * Fix BookmarksTest.kt and HomeScreenTest.kt * Fix intent matcher for SettingsAboutTest Adds a helper function for Play Store intent * Fix SettingsSearchTest.kt * ktlint fix TestHelper.kt and SettingsRobot.ktpull/543/head
parent
2899a66b15
commit
4e99fe8521
@ -0,0 +1,42 @@
|
||||
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
# Flank Documentation: https://flank.github.io/flank/
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
timeout: 30m
|
||||
async: false
|
||||
num-flaky-test-attempts: 1
|
||||
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: false
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
|
||||
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
|
||||
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
|
||||
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
|
||||
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
|
||||
|
||||
device:
|
||||
- model: Pixel2.arm
|
||||
version: 30
|
||||
locale: en_US
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
max-test-shards: 1
|
||||
num-test-runs: 1
|
||||
output-style: compact
|
||||
full-junit-result: true
|
||||
|
@ -0,0 +1,47 @@
|
||||
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
# Flank Documentation: https://flank.github.io/flank/
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
timeout: 30m
|
||||
async: false
|
||||
num-flaky-test-attempts: 2
|
||||
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: false
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest
|
||||
- class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest
|
||||
- class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest
|
||||
|
||||
device:
|
||||
- model: Pixel2.arm
|
||||
version: 28
|
||||
locale: en_US
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
max-test-shards: -1
|
||||
num-test-runs: 1
|
||||
output-style: compact
|
||||
full-junit-result: true
|
@ -0,0 +1,35 @@
|
||||
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
# Flank Documentation: https://flank.github.io/flank/
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
timeout: 30m
|
||||
async: false
|
||||
num-flaky-test-attempts: 1
|
||||
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: false
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- package org.mozilla.fenix.screenshots
|
||||
|
||||
device:
|
||||
- model: Pixel2.arm
|
||||
version: 30
|
||||
locale: en_US
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
max-test-shards: 1
|
||||
num-test-runs: 1
|
||||
output-style: compact
|
||||
full-junit-result: true
|
@ -0,0 +1,41 @@
|
||||
# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
# Flank Documentation: https://flank.github.io/flank/
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
timeout: 30m
|
||||
async: false
|
||||
num-flaky-test-attempts: 1
|
||||
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: false
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
|
||||
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
|
||||
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
|
||||
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
|
||||
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
|
||||
|
||||
device:
|
||||
- model: Pixel2.arm
|
||||
version: 30
|
||||
locale: en_US
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
max-test-shards: 1
|
||||
num-test-runs: 1
|
||||
output-style: compact
|
||||
full-junit-result: true
|
@ -1,48 +0,0 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 30m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# test and app are the only required args
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
|
||||
|
||||
device:
|
||||
- model: Pixel2
|
||||
version: 30
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# num-test-runs: the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
num-test-runs: 1
|
||||
|
@ -1,47 +0,0 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 30m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# test and app are the only required args
|
||||
app: /APP/PATH
|
||||
test: /TEST/PATH
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
device:
|
||||
- model: Pixel2
|
||||
version: 30
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test.
|
||||
max-test-shards: -1
|
||||
# repeat tests - the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
repeat-tests: 1
|
@ -1,61 +0,0 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 20m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# The number of times a test execution should be re-attempted if one or more failures occur.
|
||||
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
||||
num-flaky-test-attempts: 1
|
||||
|
||||
# test and app are the only required args
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
|
||||
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
|
||||
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
|
||||
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
|
||||
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
|
||||
|
||||
device:
|
||||
- model: Pixel2
|
||||
version: 30
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test.
|
||||
max-test-shards: 1
|
||||
# num-test-runs: the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
num-test-runs: 1
|
||||
|
@ -1,77 +0,0 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 30m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# The number of times a test execution should be re-attempted if one or more failures occur.
|
||||
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
||||
num-flaky-test-attempts: 2
|
||||
|
||||
# test and app are the only required args
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: false
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest
|
||||
- class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest
|
||||
- class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest
|
||||
|
||||
device:
|
||||
- model: Pixel2
|
||||
version: 28
|
||||
locale: en_US
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test. default: 1
|
||||
max-test-shards: -1
|
||||
# num-test-runs: the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
num-test-runs: 1
|
||||
### Output Style flag
|
||||
## Output style of execution status. May be one of [verbose, multi, single, compact].
|
||||
## For runs with only one test execution the default value is 'verbose', in other cases
|
||||
## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
|
||||
## which don't support ansi codes, to avoid corrupted output use single or verbose.
|
||||
## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
|
||||
output-style: compact
|
||||
### Full Junit Result flag
|
||||
## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
|
||||
full-junit-result: true
|
@ -1,55 +0,0 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 20m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# The number of times a test execution should be re-attempted if one or more failures occur.
|
||||
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
||||
num-flaky-test-attempts: 1
|
||||
|
||||
# test and app are the only required args
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- package org.mozilla.fenix.screenshots
|
||||
|
||||
device:
|
||||
- model: Pixel2
|
||||
version: 30
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test.
|
||||
max-test-shards: 1
|
||||
# num-test-runs: the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
num-test-runs: 1
|
||||
|
@ -1,61 +0,0 @@
|
||||
# gcloud args match the official gcloud cli
|
||||
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||
gcloud:
|
||||
results-bucket: fenix_test_artifacts
|
||||
record-video: true
|
||||
|
||||
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||
timeout: 30m
|
||||
# will start test then close socket. no reports will be generated.
|
||||
# to retrieve results later, use the "refresh" command
|
||||
# reports will be generated from /results/matrix_ids.json
|
||||
#async: true
|
||||
# will start test then leave socket open. reports will be published
|
||||
# to /results
|
||||
# see: https://github.com/TestArmada/flank/issues/339
|
||||
async: false
|
||||
|
||||
# results-history-name
|
||||
# by default, set to app name
|
||||
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||
# see: https://github.com/TestArmada/flank/issues/341
|
||||
#results-history-name: tmp_parallel
|
||||
|
||||
# The number of times a test execution should be re-attempted if one or more failures occur.
|
||||
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
||||
num-flaky-test-attempts: 1
|
||||
|
||||
# test and app are the only required args
|
||||
app: /app/path
|
||||
test: /test/path
|
||||
|
||||
auto-google-login: true
|
||||
use-orchestrator: true
|
||||
environment-variables:
|
||||
clearPackageData: true
|
||||
directories-to-pull:
|
||||
- /sdcard/screenshots
|
||||
performance-metrics: true
|
||||
|
||||
test-targets:
|
||||
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
|
||||
- class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
|
||||
- class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
|
||||
- class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
|
||||
- class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
|
||||
- class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
|
||||
|
||||
device:
|
||||
- model: Pixel2
|
||||
version: 30
|
||||
|
||||
flank:
|
||||
project: GOOGLE_PROJECT
|
||||
# test shards - the amount of groups to split the test suite into
|
||||
# set to -1 to use one shard per test.
|
||||
max-test-shards: 1
|
||||
# num-test-runs: the amount of times to run the tests.
|
||||
# 1 runs the tests once. 10 runs all the tests 10x
|
||||
num-test-runs: 1
|
||||
|
Loading…
Reference in New Issue