.gitlab-ci/tracie/tests/test.py

   1 import logging
   2 import pytest
   3 import re
   4 import shutil
   5
   6 from os import environ, chdir
   7 from os.path import dirname, exists, realpath
   8
   9 import tracie
  10
  11
  12 RESULTS_YAML = "results/results.yml"
  13 TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
  14 TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
  15 TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
  16 TRACE_PNG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace-0.png"
  17 TRACIE_DIR = dirname(realpath(__file__)) + "/.."
  18
  19 logging.basicConfig(level=logging.INFO)
  20 logger = logging.getLogger()
  21
  22
  23 def write_to(content, filename):
  24     with open(filename, 'w') as f:
  25         f.write(content)
  26
  27
  28 def read_from(filename):
  29     with open(filename) as f:
  30         content = f.read()
  31     return content
  32
  33
  34 def run_tracie():
  35     '''
  36     Run tests for the .testtrace types, using the "gl-test-device" and
  37     "vk-test-device" device names.
  38     '''
  39     result = tracie.main(["--device-name", "gl-test-device",
  40                           "--file", "./tests/traces.yml"])
  41     if not result:
  42         return False
  43     result = tracie.main(["--device-name", "vk-test-device",
  44                           "--file", "./tests/traces.yml"])
  45     return result
  46
  47
  48 def prepare_for_run(tmp_path):
  49     '''
  50     Copy all the tracie scripts to the test dir for the unit tests.
  51     This avoids polluting the normal working dir with test result artifacts.
  52     '''
  53     test_dir = str(tmp_path) + "/run"
  54     shutil.copytree(TRACIE_DIR, test_dir)
  55     # Change the working dir to the test_dir
  56     chdir(test_dir)
  57     # Set the traces-db
  58     shutil.move("./tests/test-data", "./traces-db")
  59     # Disable trace storing
  60     environ["TRACIE_STORE_IMAGES"] = "0"
  61
  62
  63 def cleanup(tmp_path):
  64     '''
  65     Performs the clean up of the test dir.
  66     '''
  67     if exists(tmp_path):
  68         shutil.rmtree(tmp_path)
  69
  70
  71 @pytest.fixture(autouse=True)
  72 def run_test(tmp_path):
  73     '''
  74     Wraps the execution of each test as follows:
  75
  76       prepare_for_run()
  77       test()
  78       cleanup()
  79     '''
  80     logger.debug("Working dir: %s", tmp_path)
  81     prepare_for_run(tmp_path)
  82     yield
  83     cleanup(tmp_path)
  84
  85
  86 def check_results_yaml_content(filename, expectations):
  87     '''
  88     Checks the content of the filename with the list of expectations
  89     passed as parameter.
  90
  91     Arguments:
  92         filename (str): The path of the file to check
  93         expectations (list): A list with the content to find in the file
  94
  95     Returns:
  96         bool: The return value. True if the content of the filename satisfies
  97               the expectations, False otherwise.
  98     '''
  99     content = read_from(filename)
 100     for e in expectations:
 101         ocurrencies = re.findall(e, content)
 102         if not len(ocurrencies):
 103             logger.error("Expectation not found in %s: %s", filename, e)
 104             return False
 105     return True
 106
 107
 108 def test_tracie_succeeds_if_all_images_match():
 109     assert run_tracie()
 110     expectations = [
 111         "actual: 5efda83854befe0155ff8517a58d5b51",
 112         "expected: 5efda83854befe0155ff8517a58d5b51",
 113     ]
 114     assert check_results_yaml_content(RESULTS_YAML, expectations)
 115
 116
 117 def test_tracie_fails_on_image_mismatch():
 118     filename = "./tests/traces.yml"
 119     content = read_from(filename)
 120     content = content.replace("5efda83854befe0155ff8517a58d5b51",
 121                               "8e0a801367e1714463475a824dab363b")
 122     write_to(content, filename)
 123     assert not run_tracie()
 124     expectations = [
 125         "actual: 5efda83854befe0155ff8517a58d5b51",
 126         "expected: 8e0a801367e1714463475a824dab363b",
 127         "trace2/test/vk-test-device/olive.testtrace-0.png"
 128     ]
 129     assert check_results_yaml_content(RESULTS_YAML, expectations)
 130
 131
 132 def test_tracie_traces_with_and_without_checksum():
 133     filename = "./tests/traces.yml"
 134     content = read_from(filename)
 135     content += '''  - path: trace1/red.testtrace
 136     expectations:
 137     - device: bla
 138       checksum: 000000000000000'''
 139     write_to(content, filename)
 140
 141     # red.testtrace should be skipped, since it doesn't
 142     # have any checksums for our device
 143     filename = "./traces-db/trace1/red.testtrace"
 144     content = "ff0000ff"
 145     write_to(content, filename)
 146     assert run_tracie()
 147
 148
 149 def test_tracie_only_traces_without_checksum():
 150     filename = "./tests/traces.yml"
 151     content = '''traces:
 152   - path: trace1/red.testtrace
 153     expectations:
 154     - device: bla
 155       checksum: 000000000000000'''
 156     write_to(content, filename)
 157
 158     # red.testtrace should be skipped, since it doesn't
 159     # have any checksums for our device
 160     filename = "./traces-db/trace1/red.testtrace"
 161     content = "ff0000ff"
 162     write_to(content, filename)
 163     assert run_tracie()
 164
 165
 166 def test_tracie_with_no_traces():
 167     filename = "./tests/traces.yml"
 168     content = 'traces:'
 169     write_to(content, filename)
 170     assert run_tracie()
 171     expectations = [
 172         "{}",
 173     ]
 174     assert check_results_yaml_content(RESULTS_YAML, expectations)
 175
 176
 177 def test_tracie_fails_on_dump_image_error():
 178     # "invalid" should fail to parse as rgba and
 179     # cause an error
 180     filename = "./traces-db/trace1/magenta.testtrace"
 181     write_to("invalid\n", filename)
 182     run_tracie()
 183     expectations = [
 184         "actual: error",
 185         "expected: 8e0a801367e1714463475a824dab363b",
 186         "trace1/magenta.testtrace",
 187     ]
 188     assert check_results_yaml_content(RESULTS_YAML, expectations)
 189
 190
 191 def test_tracie_stores_only_logs_on_checksum_match():
 192     assert run_tracie()
 193     assert exists(TRACE_LOG_TEST1)
 194     assert exists(TRACE_LOG_TEST2)
 195     assert not exists(TRACE_PNG_TEST1)
 196     assert not exists(TRACE_PNG_TEST2)
 197
 198
 199 def test_tracie_stores_images_on_checksum_mismatch():
 200     filename = "./tests/traces.yml"
 201     content = read_from(filename)
 202     content = content.replace("5efda83854befe0155ff8517a58d5b51",
 203                               "8e0a801367e1714463475a824dab363b")
 204     write_to(content, filename)
 205     assert not run_tracie()
 206     assert not exists(TRACE_PNG_TEST1)
 207     assert exists(TRACE_PNG_TEST2)
 208
 209
 210 def test_tracie_stores_images_on_request():
 211     environ["TRACIE_STORE_IMAGES"] = "1"
 212     assert run_tracie()
 213     assert exists(TRACE_PNG_TEST1)
 214     assert exists(TRACE_PNG_TEST2)