From 550a4f77648f68bc48df34ce9d39dc3df1461f13 Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Mon, 4 May 2020 18:11:08 +0200 Subject: [PATCH] ci: Migrate tracie tests done in shell script to pytest v2: Verbatim translation from the original shell script Make the corrections visible in explicit commits (Andres) Remove redundant code (Alexandros) Code style nitpick (Rohan) Reimplementation of the tracie's self-tests using a pythonic test suit (pytest). The new tracie/test.py module is almost a direct translation of the tests defined in the tracie/test.sh. This new implementation of the test provides a more common framework where define the tests. Also allows a better introspection for the tests results and/or resulting errors. This patch also adds python3-pytest as dependency for the built images and adapts the tracie-runner scripts to run the self-test using pytest. Signed-off-by: Pablo Saavedra Reviewed-by: Alexandros Frantzis [v1] Reviewed-by: Andres Gomez Reviewed-by: Rohan Garg [v1] Part-of: --- .gitlab-ci.yml | 6 +- .gitlab-ci/container/x86_test-gl.sh | 1 + .gitlab-ci/container/x86_test-vk.sh | 1 + .gitlab-ci/tracie-runner-gl.sh | 2 +- .gitlab-ci/tracie-runner-vk.sh | 2 +- .gitlab-ci/tracie/tests/__init__.py | 0 .gitlab-ci/tracie/tests/test.py | 186 ++++++++++++++++++++++++++++ .gitlab-ci/tracie/tests/test.sh | 154 ----------------------- 8 files changed, 193 insertions(+), 159 deletions(-) create mode 100644 .gitlab-ci/tracie/tests/__init__.py create mode 100644 .gitlab-ci/tracie/tests/test.py delete mode 100755 .gitlab-ci/tracie/tests/test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 710934a0a7a..62dccc6f34a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -136,7 +136,7 @@ x86_build: - .fdo.container-build@debian - .container variables: - FDO_DISTRIBUTION_TAG: &x86_build "2020-05-08" + FDO_DISTRIBUTION_TAG: &x86_build "2020-05-18" .use-x86_build: variables: @@ -149,13 +149,13 @@ x86_build: x86_test-gl: extends: x86_build variables: - FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-08" + FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-18" # Debian 10 based x86 test image for VK x86_test-vk: extends: x86_build variables: - FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-08" + FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-18" # Debian 9 based x86 build image (old LLVM) x86_build_old: diff --git a/.gitlab-ci/container/x86_test-gl.sh b/.gitlab-ci/container/x86_test-gl.sh index a8220df0ec4..41816bb99b6 100644 --- a/.gitlab-ci/container/x86_test-gl.sh +++ b/.gitlab-ci/container/x86_test-gl.sh @@ -64,6 +64,7 @@ apt-get install -y --no-remove \ python3-mako \ python3-numpy \ python3-pil \ + python3-pytest \ python3-requests \ python3-six \ python3-yaml \ diff --git a/.gitlab-ci/container/x86_test-vk.sh b/.gitlab-ci/container/x86_test-vk.sh index 455506d2972..0c8983aa09f 100644 --- a/.gitlab-ci/container/x86_test-vk.sh +++ b/.gitlab-ci/container/x86_test-vk.sh @@ -64,6 +64,7 @@ apt-get install -y --no-remove \ python \ python3-distutils \ python3-pil \ + python3-pytest \ python3-requests \ python3-yaml \ vulkan-tools \ diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh index a8dc8206d05..426333ce295 100755 --- a/.gitlab-ci/tracie-runner-gl.sh +++ b/.gitlab-ci/tracie-runner-gl.sh @@ -26,7 +26,7 @@ RESULTS=`pwd`/results mkdir -p $RESULTS # Perform a self-test to ensure tracie is working properly. -"$INSTALL/tracie/tests/test.sh" +python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py if [ "$GALLIUM_DRIVER" = "virpipe" ]; then # tracie is to use virpipe, and virgl_test_server llvmpipe diff --git a/.gitlab-ci/tracie-runner-vk.sh b/.gitlab-ci/tracie-runner-vk.sh index a3e13f6c1b4..f046b7c9b40 100755 --- a/.gitlab-ci/tracie-runner-vk.sh +++ b/.gitlab-ci/tracie-runner-vk.sh @@ -21,7 +21,7 @@ export DXVK_LOG_LEVEL="none" export DXVK_STATE_CACHE=0 # Perform a self-test to ensure tracie is working properly. -"$INSTALL/tracie/tests/test.sh" +python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py # Sanity check to ensure that our environment is sufficient to make our tests # run against the Mesa built by CI, rather than any installed distro version. diff --git a/.gitlab-ci/tracie/tests/__init__.py b/.gitlab-ci/tracie/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.gitlab-ci/tracie/tests/test.py b/.gitlab-ci/tracie/tests/test.py new file mode 100644 index 00000000000..f8c55cae985 --- /dev/null +++ b/.gitlab-ci/tracie/tests/test.py @@ -0,0 +1,186 @@ +import logging +import pytest +import re +import shutil + +from os import environ, chdir +from os.path import dirname, exists, realpath + +import tracie + + +RESULTS_YAML = "results/results.yml" +TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log" +TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log" +TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png" +TRACE_PNG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace-0.png" +TRACIE_DIR = dirname(realpath(__file__)) + "/.." + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger() + + +def write_to(content, filename): + with open(filename, 'w') as f: + f.write(content) + + +def read_from(filename): + with open(filename) as f: + content = f.read() + return content + + +def run_tracie(): + ''' + Run tests for the .testtrace types, using the "gl-test-device" and + "vk-test-device" device names. + ''' + result = tracie.main(["--device-name", "gl-test-device", + "--file", "./tests/traces.yml"]) + if not result: + return False + result = tracie.main(["--device-name", "vk-test-device", + "--file", "./tests/traces.yml"]) + return result + + +def prepare_for_run(tmp_path): + ''' + Copy all the tracie scripts to the test dir for the unit tests. + This avoids polluting the normal working dir with test result artifacts. + ''' + test_dir = str(tmp_path) + "/run" + shutil.copytree(TRACIE_DIR, test_dir) + # Change the working dir to the test_dir + chdir(test_dir) + # Set the traces-db + shutil.move("./tests/test-data", "./traces-db") + # Disable trace storing + environ["TRACIE_STORE_IMAGES"] = "0" + + +def cleanup(tmp_path): + ''' + Performs the clean up of the test dir. + ''' + if exists(tmp_path): + shutil.rmtree(tmp_path) + + +@pytest.fixture(autouse=True) +def run_test(tmp_path): + ''' + Wraps the execution of each test as follows: + + prepare_for_run() + test() + cleanup() + ''' + logger.debug("Working dir: %s", tmp_path) + prepare_for_run(tmp_path) + yield + cleanup(tmp_path) + + +def check_results_yaml_content(filename, expectations): + ''' + Checks the content of the filename with the list of expectations + passed as parameter. + + Arguments: + filename (str): The path of the file to check + expectations (list): A list with the content to find in the file + + Returns: + bool: The return value. True if the content of the filename satisfies + the expectations, False otherwise. + ''' + content = read_from(filename) + for e in expectations: + ocurrencies = re.findall(e, content) + if not len(ocurrencies): + logger.error("Expectation not found in %s: %s", filename, e) + return False + return True + + +def test_tracie_succeeds_if_all_images_match(): + assert run_tracie() + expectations = [ + "actual: 5efda83854befe0155ff8517a58d5b51", + "expected: 5efda83854befe0155ff8517a58d5b51", + ] + assert check_results_yaml_content(RESULTS_YAML, expectations) + + +def test_tracie_fails_on_image_mismatch(): + filename = "./tests/traces.yml" + content = read_from(filename) + content = content.replace("5efda83854befe0155ff8517a58d5b51", + "8e0a801367e1714463475a824dab363b") + write_to(content, filename) + assert not run_tracie() + expectations = [ + "actual: 5efda83854befe0155ff8517a58d5b51", + "expected: 8e0a801367e1714463475a824dab363b", + "trace2/test/vk-test-device/olive.testtrace-0.png" + ] + assert check_results_yaml_content(RESULTS_YAML, expectations) + + +def test_tracie_skips_traces_without_checksum(): + filename = "./tests/traces.yml" + content = '''traces: + - path: trace1/red.testtrace + expectations: + - device: bla + checksum: 000000000000000''' + write_to(content, filename) + + # red.testtrace should be skipped, since it doesn't + # have any checksums for our device + filename = "./traces-db/trace1/red.testtrace" + content = "ff0000ff" + write_to(content, filename) + assert run_tracie() + + +def test_tracie_fails_on_dump_image_error(): + # "invalid" should fail to parse as rgba and + # cause an error + filename = "./traces-db/trace1/magenta.testtrace" + write_to("invalid\n", filename) + run_tracie() + expectations = [ + "actual: error", + "expected: 8e0a801367e1714463475a824dab363b", + "trace1/magenta.testtrace", + ] + assert check_results_yaml_content(RESULTS_YAML, expectations) + + +def test_tracie_stores_only_logs_on_checksum_match(): + assert run_tracie() + assert exists(TRACE_LOG_TEST1) + assert exists(TRACE_LOG_TEST2) + assert not exists(TRACE_PNG_TEST1) + assert not exists(TRACE_PNG_TEST2) + + +def test_tracie_stores_images_on_checksum_mismatch(): + filename = "./tests/traces.yml" + content = read_from(filename) + content = content.replace("5efda83854befe0155ff8517a58d5b51", + "8e0a801367e1714463475a824dab363b") + write_to(content, filename) + assert not run_tracie() + assert not exists(TRACE_PNG_TEST1) + assert exists(TRACE_PNG_TEST2) + + +def test_tracie_stores_images_on_request(): + environ["TRACIE_STORE_IMAGES"] = "1" + assert run_tracie() + assert exists(TRACE_PNG_TEST1) + assert exists(TRACE_PNG_TEST2) diff --git a/.gitlab-ci/tracie/tests/test.sh b/.gitlab-ci/tracie/tests/test.sh deleted file mode 100755 index 7dbcfce3241..00000000000 --- a/.gitlab-ci/tracie/tests/test.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/sh - -TRACIE_DIR="$(dirname "$(readlink -f "$0")")/.." -TEST_DIR="" -TEST_EXIT=0 - -assert() { - if ! $1; then - echo "Assertion failed: \"$1\"" - exit 1 - fi -} - -run_tracie() { - # Run tests for the .testtrace types, using the "gl-test-device" and "vk-test-device" device names. - python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name gl-test-device && \ - python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name vk-test-device -} - -cleanup() { - [ "$TEST_DIR" = "/tmp/*" ] && rm -rf "$TEST_DIR" -} - -prepare_for_run() { - TEST_DIR="$(mktemp -d -t tracie.test.XXXXXXXXXX)" - # Copy all the tracie scripts to the test dir for the run-tests.sh script. - # This avoids polluting the normal working dir with test result artifacts. - cp -R "$TRACIE_DIR"/. "$TEST_DIR" - cd "$TEST_DIR" - mkdir traces-db - mv tests/test-data/* traces-db/. - trap cleanup EXIT - # Ensure we have a clean environment. - unset TRACIE_STORE_IMAGES -} - -run_test() { - prepare_for_run - log=$(mktemp) - if ($1 > "$log" 2>&1 ;); then - if [ -t 1 ]; then - echo "$1: \e[0;32mSuccess\e[0m" - else - echo "$1: Success" - fi - else - if [ -t 1 ]; then - echo "$1: \e[0;31mFail\e[0m" - else - echo "$1: Fail" - fi - cat "$log" - TEST_EXIT=1 - fi - rm "$log" - cleanup -} - -assert_results_yaml_contains() { - grep -q "actual: $1" $4 - assert "[ $? = 0 ]" - - grep -q "expected: $2" $4 - assert "[ $? = 0 ]" - - if [ $3 != "" ]; then - grep -q $3 $4 - fi - - assert "[ $? = 0 ]" -} - -tracie_succeeds_if_all_images_match() { - run_tracie - assert "[ $? = 0 ]" - assert_results_yaml_contains 5efda83854befe0155ff8517a58d5b51 5efda83854befe0155ff8517a58d5b51 "" "$PWD/results/results.yml" -} - -tracie_fails_on_image_mismatch() { - sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \ - "$TEST_DIR/tests/traces.yml" - - run_tracie - assert "[ $? != 0 ]" - assert_results_yaml_contains 5efda83854befe0155ff8517a58d5b51 8e0a801367e1714463475a824dab363b "trace2/test/vk-test-device/olive.testtrace-0.png" "$PWD/results/results.yml" -} - -tracie_skips_traces_without_checksum() { - echo "traces:" > "$TEST_DIR/tests/traces.yml" - echo " - path: trace1/red.testtrace" >> "$TEST_DIR/tests/traces.yml" - echo " expectations:" >> "$TEST_DIR/tests/traces.yml" - echo " - device: bla" >> "$TEST_DIR/tests/traces.yml" - echo " checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml" - # red.testtrace should be skipped, since it doesn't - # have any checksums for our device - echo "ff0000ff" > traces-db/trace1/red.testtrace - - run_tracie - assert "[ $? = 0 ]" -} - -tracie_fails_on_dump_image_error() { - # "invalid" should fail to parse as rgba and - # cause an error - echo "invalid" > traces-db/trace1/magenta.testtrace - - run_tracie - assert "[ $? != 0 ]" - assert_results_yaml_contains "error" "8e0a801367e1714463475a824dab363b" "trace1/magenta.testtrace" "$PWD/results/results.yml" -} - -tracie_stores_only_logs_on_checksum_match() { - run_tracie - assert "[ $? = 0 ]" - - assert "[ -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace.log" ]" - assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace.log" ]" - - assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]" - assert "[ ! -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]" - - ls -lR "$TEST_DIR" -} - -tracie_stores_images_on_checksum_mismatch() { - sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \ - "$TEST_DIR/tests/traces.yml" - - run_tracie - assert "[ $? != 0 ]" - - assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]" - assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]" -} - -tracie_stores_images_on_request() { - (export TRACIE_STORE_IMAGES=1; run_tracie) - assert "[ $? = 0 ]" - - assert "[ -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]" - assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]" - - ls -lR "$TEST_DIR" -} - -run_test tracie_succeeds_if_all_images_match -run_test tracie_fails_on_image_mismatch -run_test tracie_skips_traces_without_checksum -run_test tracie_fails_on_dump_image_error -run_test tracie_stores_only_logs_on_checksum_match -run_test tracie_stores_images_on_checksum_mismatch -run_test tracie_stores_images_on_request - -exit $TEST_EXIT -- 2.30.2