From 7859eb13904d371d7416368ed952987b34910586 Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Thu, 12 Sep 2019 12:34:50 -0700 Subject: [PATCH] gitlab-ci: Make the test job fail when bugs are unexpectedly fixed. If people fix bugs without updating the expected-fails list, then we end up with a lack of coverage of those failures in the future. Also, some day down the line another developer ends up trying to figure out if the bug was actually fixed or their environment is just failing to reproduce it. Suggested-by: Kristian H. Kristensen Reviewed-by: Adam Jackson Acked-by: Rob Clark Reviewed-by: Kristian H. Kristensen --- .gitlab-ci/deqp-runner.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh index bb9c7c80d56..822665ca66e 100755 --- a/.gitlab-ci/deqp-runner.sh +++ b/.gitlab-ci/deqp-runner.sh @@ -123,3 +123,14 @@ if [ -s /tmp/new-fails.txt ]; then else echo "No new failures" fi + +sort /tmp/case-list.txt > /tmp/sorted-case-list.txt +comm -12 /tmp/sorted-case-list.txt /tmp/expected-fails.txt > /tmp/expected-fails-in-caselist.txt +comm -13 $RESULTS/fails.txt /tmp/expected-fails-in-caselist.txt > /tmp/new-passes.txt +if [ -s /tmp/new-passes.txt ]; then + echo "Unexpected passes, please update $DEQP_EXPECTED_FAILS (or add flaky tests to $DEQP_SKIPS):" + cat /tmp/new-passes.txt + exit 1 +else + echo "No new passes" +fi -- 2.30.2