From 226658cfb10dde4bea60611519f2f769604ee369 Mon Sep 17 00:00:00 2001 From: Jackie Johnson <107960801+jjSDET@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:34:04 -0600 Subject: [PATCH] Bug 1867539 - Add UI Test Tooling Activation --- .../androidTest/lib/testrail_conn.py | 102 ++++++++++++ .../taskcluster/androidTest/testrail.py | 154 ++++++++++++++++++ automation/taskcluster/androidTest/ui-test.sh | 10 +- 3 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 automation/taskcluster/androidTest/lib/testrail_conn.py create mode 100644 automation/taskcluster/androidTest/testrail.py diff --git a/automation/taskcluster/androidTest/lib/testrail_conn.py b/automation/taskcluster/androidTest/lib/testrail_conn.py new file mode 100644 index 000000000..41d1f2667 --- /dev/null +++ b/automation/taskcluster/androidTest/lib/testrail_conn.py @@ -0,0 +1,102 @@ +# flake8: noqa +"""TestRail API binding for Python 3.x. + +(API v2, available since TestRail 3.0) + +Compatible with TestRail 3.0 and later. + +Learn more: + +http://docs.gurock.com/testrail-api2/start +http://docs.gurock.com/testrail-api2/accessing + +Copyright Gurock Software GmbH. See license.md for details. +""" + +import base64 +import json + +import requests + + +class APIClient: + def __init__(self, base_url): + self.user = '' + self.password = '' + if not base_url.endswith('/'): + base_url += '/' + self.__url = base_url + 'index.php?/api/v2/' + + def send_get(self, uri, filepath=None): + """Issue a GET request (read) against the API. + + Args: + uri: The API method to call including parameters, e.g. get_case/1. + filepath: The path and file name for attachment download; used only + for 'get_attachment/:attachment_id'. + + Returns: + A dict containing the result of the request. + """ + return self.__send_request('GET', uri, filepath) + + def send_post(self, uri, data): + """Issue a POST request (write) against the API. + + Args: + uri: The API method to call, including parameters, e.g. add_case/1. + data: The data to submit as part of the request as a dict; strings + must be UTF-8 encoded. If adding an attachment, must be the + path to the file. + + Returns: + A dict containing the result of the request. + """ + return self.__send_request('POST', uri, data) + + def __send_request(self, method, uri, data): + url = self.__url + uri + + auth = str( + base64.b64encode( + bytes('%s:%s' % (self.user, self.password), 'utf-8') + ), + 'ascii' + ).strip() + headers = {'Authorization': 'Basic ' + auth} + + if method == 'POST': + if uri[:14] == 'add_attachment': # add_attachment API method + files = {'attachment': (open(data, 'rb'))} + response = requests.post(url, headers=headers, files=files) + files['attachment'].close() + else: + headers['Content-Type'] = 'application/json' + payload = bytes(json.dumps(data), 'utf-8') + response = requests.post(url, headers=headers, data=payload) + else: + headers['Content-Type'] = 'application/json' + response = requests.get(url, headers=headers) + + if response.status_code > 201: + try: + error = response.json() + except requests.exceptions.HTTPError: # response.content not formatted as JSON + error = str(response.content) + raise APIError('TestRail API returned HTTP %s (%s)' % (response.status_code, error)) + else: + if uri[:15] == 'get_attachment/': # Expecting file, not JSON + try: + open(data, 'wb').write(response.content) + return (data) + except FileNotFoundError: + return ("Error saving attachment.") + else: + try: + return response.json() + except requests.exceptions.HTTPError: + return {} + + +class APIError(Exception): + pass diff --git a/automation/taskcluster/androidTest/testrail.py b/automation/taskcluster/androidTest/testrail.py new file mode 100644 index 000000000..b75efa7c3 --- /dev/null +++ b/automation/taskcluster/androidTest/testrail.py @@ -0,0 +1,154 @@ +""" +This Python script is designed to automate the process of creating milestones +and test runs in TestRail, and updating test cases based on the results of +automated smoke tests for different product releases. + +Below is a summary of its functionality in order of execution: + +1. Environment and Credentials Setup: + - Imports necessary libraries and modules. + - Loads environmental variables from "execution_metadata.env". + - Reads and processes TestRail credentials from '.testrail_credentials.json'. + +2. Environment Variables Validation: + - Retrieves and validates several environment variables like `PRODUCT_TYPE`, + `RELEASE_TYPE`, `VERSION_NUMBER`, and `TEST_STATUS`. + - Ensures `TEST_STATUS` is either 'PASS' or 'FAIL'. + +3. Utility Functions: + - `parse_release_number()`: Parses the version number to extract a specific part. + - `build_milestone_name()`: Constructs a milestone name based on product type, + release type, and version number. + - `build_milestone_description()`: Creates a detailed description for the milestone + including the current date and placeholders for various testing statuses. + +4. TestRail Integration: + - Defines a `TestRail` class that handles interactions with the TestRail API. + - Includes methods to create milestones, create test runs, and update test cases. + +5. Main Execution: + - Checks if `TEST_STATUS` is 'PASS'. If not, it raises an error to trigger a Slack notification. + - Sets parameters for a demo TestRail project. + - Instantiates the `TestRail` class. + - Creates a milestone in TestRail and retrieves its ID. + - Creates test runs for each device/API combination (currently hardcoded for phase 1 testing) + and updates test cases to 'passed' status. + +6. Phase 1 and Phase 2 Notes: + - The script is currently in Phase 1, where certain values are hardcoded for testing. + - In Phase 2, these hardcoded values will be parameterized for broader usage. +""" + + +import json +import os +import textwrap +from lib.testrail_conn import APIClient +from dotenv import load_dotenv +from datetime import datetime + +try: + load_dotenv("execution_metadata.env") # Attempt to load .env file +except FileNotFoundError: + raise FileNotFoundError("The .env file was not found.") +except Exception as e: + raise Exception(f"An error occurred while loading the .env file: {e}") + +try: + with open('.testrail_credentials.json', 'r') as file: + secret = json.load(file) + TESTRAIL_HOST = secret['host'] + TESTRAIL_USERNAME = secret['username'] + TESTRAIL_PASSWORD = secret['password'] +except json.JSONDecodeError as e: + raise ValueError("Failed to load testrail credentials : {e}") + +try: + PRODUCT_TYPE = os.environ["PRODUCT_TYPE"] + RELEASE_TYPE = os.environ["RELEASE_TYPE"] + VERSION_NUMBER = os.environ["MOBILE_HEAD_REF"] + TEST_STATUS = os.environ["TEST_STATUS"] + + if TEST_STATUS not in ('PASS', 'FAIL'): + raise ValueError(f"ERROR: Invalid TEST_STATUS value: {TEST_STATUS}") +except KeyError as e: + raise ValueError(f"ERROR: Missing Environment Variable: {e}") + +def parse_release_number(VERSION_NUMBER): + parts = VERSION_NUMBER.split('_') + return parts[1] + +def build_milestone_name(product_type, release_type, version_number): + return f"Automated smoke testing sign-off - {product_type} {release_type} {version_number}" + +def build_milestone_description(milestone_name): + current_date = datetime.now() + formatted_date = current_date = current_date.strftime("%B %d, %Y") + return textwrap.dedent(f""" + RELEASE: {milestone_name}\n\n\ + RELEASE_TAG_URL: https://github.com/mozilla-mobile/firefox-android/releases\n\n\ + RELEASE_DATE: {formatted_date}\n\n\ + TESTING_STATUS: [ TBD ]\n\n\ + QA_RECOMMENDATION:[ TBD ]\n\n\ + QA_RECOMENTATION_VERBOSE: \n\n\ + TESTING_SUMMARY\n\n\ + Known issues: n/a\n\ + New issue: n/a\n\ + Verified issue: + """) + +class TestRail(): + + def __init__(self): + try: + self.client = APIClient(TESTRAIL_HOST) + self.client.user = TESTRAIL_USERNAME + self.client.password = TESTRAIL_PASSWORD + except KeyError as e: + raise ValueError(f"ERROR: Missing Testrail Env Var: {e}") + + # Public Methods + + def create_milestone(self, testrail_project_id, title, description): + data = {"name": title, "description": description} + return self.client.send_post(f'add_milestone/{testrail_project_id}', data) + + def create_test_run(self, testrail_project_id, testrail_milestone_id, name_run, testrail_suite_id): + data = {"name": name_run, "milestone_id": testrail_milestone_id, "suite_id": testrail_suite_id} + return self.client.send_post(f'add_run/{testrail_project_id}', data) + + def update_test_cases_to_passed(self, testrail_project_id, testrail_run_id, testrail_suite_id): + test_cases = self._get_test_cases(testrail_project_id, testrail_suite_id) + data = { "results": [{"case_id": test_case['id'], "status_id": 1} for test_case in test_cases]} + return testrail._update_test_run_results(testrail_run_id, data) + + # Private Methods + + def _get_test_cases(self, testrail_project_id, testrail_test_suite_id): + return self.client.send_get(f'get_cases/{testrail_project_id}&suite_id={testrail_test_suite_id}') + + def _update_test_run_results(self, testrail_run_id, data): + return self.client.send_post(f'add_results_for_cases/{testrail_run_id}', data) + +if __name__ == "__main__": + if TEST_STATUS != 'PASS': + raise ValueError("Tests failed. Sending Slack Notification....") + + # There are for a dummy Testrail project used for Phase 1 testing of this script + # They will be parameterized during Phase 2 of script hardening + PROJECT_ID = 53 # Firefox for FireTV + TEST_SUITE_ID = 45442 # Demo Test Suite + + testrail = TestRail() + milestone_name = build_milestone_name(PRODUCT_TYPE, RELEASE_TYPE, parse_release_number(VERSION_NUMBER)) + milestone_description = build_milestone_description(milestone_name) + + # Create milestone for 'Firefox for FireTV' and store the ID + milestone_id = testrail.create_milestone(PROJECT_ID, milestone_name, milestone_description)['id'] + + # Create test run for each Device/API and update test cases to 'passed' + # The Firebase Test devices are temporarily hard-coded during testing + # and will be parameterized in Phase 2 of hardening + for test_run_name in ['Google Pixel 32(Android11)', 'Google Pixel2(Android9)']: + test_run_id = testrail.create_test_run(PROJECT_ID, milestone_id, test_run_name, TEST_SUITE_ID)['id'] + testrail.update_test_cases_to_passed(PROJECT_ID, test_run_id, TEST_SUITE_ID) diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index a0498d8f5..a4df3971b 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -102,10 +102,18 @@ function failure_check() { echo if [[ $exitcode -ne 0 ]]; then echo "FAILURE: UI test run failed, please check above URL" + TEST_STATUS="FAIL" else - echo "All UI test(s) have passed!" + echo "All UI test(s) have passed!" + TEST_STATUS="PASS" fi + { + echo "TEST_STATUS=${TEST_STATUS}" + echo "PRODUCT_TYPE=${PRODUCT_TYPE}" + echo "RELEASE_TYPE=${RELEASE_TYPE}" + } >> execution_metadata.env + echo echo "RESULTS" echo