gitlab-ci: Run the GLES2 CTS on llvmpipe.

This is the start of doing CTS tests on merges to Mesa master.  We use
the surfaceless platform so that we don't need to bother bringing up
weston or X11.  The surface size is kept low to reduce runtime, but
this comes at the cost of many rendering tests skipping due to
too-small render targets (as we see the impact of Mesa on the shared
runner pool, we can reevaluate this and what set of CTS tests we want
to run).

We split the job up across 4 runners (each at 4 llvmpipe threads), so
that the job can load-balance across our shared runners and finish
sooner (since dEQP is very single-thread-performance bound).

Reviewed-by: Eric Engestrom <eric.engestrom@intel.com>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aca5d7624d08238d89fcdc01ca47c7b67eaad314..22aa424b7aedf5abb8dde50696a8870c1aaa7a6c 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,6 +26,7 @@ include:
 stages:
   - containers-build
   - build+test
+  - test
 
 
 # When to automatically run the CI
@@ -39,6 +40,14 @@ stages:
     when:
       - runner_system_failure
 
+.ci-deqp-artifacts: &ci-deqp-artifacts
+  artifacts:
+    when: always
+    untracked: false
+    paths:
+      # Watch out!  Artifacts are relative to the build dir.
+      # https://gitlab.com/gitlab-org/gitlab-ce/commit/8788fb925706cad594adf6917a6c5f6587dd1521
+      - artifacts
 
 # CONTAINERS
 
@@ -77,6 +86,12 @@ debian:
     - ccache --zero-stats || true
     - ccache --show-stats || true
   after_script:
+    # In case the install dir is being saved as artifacts, tar it up
+    # so that symlinks and hardlinks aren't each packed separately in
+    # the zip file.
+    - if [ -d install ]; then
+        tar -cf artifacts/install.tar install;
+      fi
     - export CCACHE_DIR="$PWD/ccache"
     - ccache --show-stats
 
@@ -176,6 +191,7 @@ meson-main:
       -D tools=all
     MESON_SHADERDB: "true"
     BUILDTYPE: "debugoptimized"
+  <<: *ci-deqp-artifacts
 
 meson-clover:
   extends: .meson-build
@@ -247,6 +263,7 @@ meson-vulkan:
       -D gallium-xa=false
       -D gallium-nine=false
       -D llvm=false
+  <<: *ci-deqp-artifacts
   script:
     - .gitlab-ci/meson-build.sh
 
@@ -306,3 +323,32 @@ scons-llvm:
     LLVM_VERSION: "3.4"
     # LLVM 3.4 packages were built with an old libstdc++ ABI
     CXX: "g++ -D_GLIBCXX_USE_CXX11_ABI=0"
+
+.deqp-test:
+  <<: *ci-run-policy
+  stage: test
+  image: $DEBIAN_IMAGE
+  variables:
+    GIT_STRATEGY: none # testing doesn't build anything from source
+    DEQP_SKIPS: deqp-default-skips.txt
+  script:
+    # Note: Build dir (and thus install) may be dirty due to GIT_STRATEGY
+    - rm -rf install
+    - tar -xf artifacts/install.tar
+    - ./artifacts/deqp-runner.sh
+  artifacts:
+    when: on_failure
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - results/
+
+test-llvmpipe-gles2:
+  parallel: 4
+  variables:
+    DEQP_VER: gles2
+    DEQP_EXPECTED_FAILS: deqp-llvmpipe-fails.txt
+    LIBGL_ALWAYS_SOFTWARE: "true"
+    DEQP_RENDERER_MATCH: "llvmpipe"
+  extends: .deqp-test
+  dependencies:
+    - meson-main

diff --git a/.gitlab-ci/deqp-default-skips.txt b/.gitlab-ci/deqp-default-skips.txt
new file mode 100644 (file)
index 0000000..70dc0cb
--- /dev/null
+++ b/.gitlab-ci/deqp-default-skips.txt
@@ -0,0 +1,10 @@
+# Note: skips lists for CI are just a list of lines that, when
+# non-zero-length and not starting with '#', will regex match to
+# delete lines from the test list.  Be careful.
+
+# Skip the perf/stress tests to keep runtime manageable
+dEQP-GLES[0-9]*.performance
+dEQP-GLES[0-9]*.stress
+
+# These are really slow on tiling architectures (including llvmpipe).
+dEQP-GLES[0-9]*.functional.flush_finish

diff --git a/.gitlab-ci/deqp-llvmpipe-fails.txt b/.gitlab-ci/deqp-llvmpipe-fails.txt
new file mode 100644 (file)
index 0000000..05da36f
--- /dev/null
+++ b/.gitlab-ci/deqp-llvmpipe-fails.txt
@@ -0,0 +1,124 @@
+dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center
+dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner
+dEQP-GLES2.functional.clipping.point.wide_point_clip
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_center
+dEQP-GLES2.functional.clipping.point.wide_point_clip_viewport_corner
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_two.clip_neg_y_neg_z_and_neg_x_neg_y_pos_z
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_two.clip_pos_y_pos_z_and_neg_x_neg_y_neg_z
+dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4
+dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_depth_component16
+dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_stencil_index8
+dEQP-GLES2.functional.fbo.render.depth.rbo_rgba4_depth_component16
+dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgba4
+dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.no_rebind_rbo_rgba4_stencil_index8
+dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgba4
+dEQP-GLES2.functional.fbo.render.recreate_colorbuffer.rebind_rbo_rgba4_stencil_index8
+dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.no_rebind_rbo_rgba4_depth_component16
+dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.rebind_rbo_rgba4_depth_component16
+dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.no_rebind_rbo_rgba4_stencil_index8
+dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.rebind_rbo_rgba4_stencil_index8
+dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4
+dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4_depth_component16
+dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgba4_depth_component16
+dEQP-GLES2.functional.polygon_offset.default_displacement_with_units
+dEQP-GLES2.functional.polygon_offset.fixed16_displacement_with_units
+dEQP-GLES2.functional.rasterization.interpolation.basic.line_loop_wide
+dEQP-GLES2.functional.rasterization.interpolation.basic.line_strip_wide
+dEQP-GLES2.functional.rasterization.interpolation.basic.lines_wide
+dEQP-GLES2.functional.rasterization.interpolation.projected.line_loop_wide
+dEQP-GLES2.functional.rasterization.interpolation.projected.line_strip_wide
+dEQP-GLES2.functional.rasterization.interpolation.projected.lines_wide
+dEQP-GLES2.functional.rasterization.limits.points
+dEQP-GLES2.functional.shaders.texture_functions.fragment.texture2d_bias
+dEQP-GLES2.functional.shaders.texture_functions.fragment.texture2dproj_vec3_bias
+dEQP-GLES2.functional.shaders.texture_functions.fragment.texture2dproj_vec4_bias
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_clamp_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_mirror_etc1
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_mirror_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_repeat_etc1
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_linear_repeat_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_clamp_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_mirror_etc1
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_mirror_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_etc1
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_l8
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_rgb888
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_rgba4444
+dEQP-GLES2.functional.texture.filtering.2d.linear_mipmap_linear_nearest_repeat_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_clamp_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_mirror_etc1
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_mirror_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_repeat_etc1
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_repeat_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_clamp_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_mirror_etc1
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_mirror_rgba8888
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_etc1
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_l8
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_rgb888
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_rgba4444
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_nearest_repeat_rgba8888
+dEQP-GLES2.functional.texture.mipmap.2d.affine.linear_linear_repeat
+dEQP-GLES2.functional.texture.mipmap.2d.affine.nearest_linear_clamp
+dEQP-GLES2.functional.texture.mipmap.2d.affine.nearest_linear_mirror
+dEQP-GLES2.functional.texture.mipmap.2d.affine.nearest_linear_repeat
+dEQP-GLES2.functional.texture.mipmap.2d.basic.linear_linear_repeat
+dEQP-GLES2.functional.texture.mipmap.2d.basic.linear_linear_repeat_non_square
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_clamp
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_clamp_non_square
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_mirror
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_mirror_non_square
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_repeat
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_repeat_non_square
+dEQP-GLES2.functional.texture.mipmap.2d.projected.linear_linear_repeat
+dEQP-GLES2.functional.texture.mipmap.2d.projected.nearest_linear_clamp
+dEQP-GLES2.functional.texture.mipmap.2d.projected.nearest_linear_mirror
+dEQP-GLES2.functional.texture.mipmap.2d.projected.nearest_linear_repeat
+dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_linear
+dEQP-GLES2.functional.texture.mipmap.cube.basic.linear_nearest
+dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_linear
+dEQP-GLES2.functional.texture.mipmap.cube.bias.linear_nearest
+dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_linear
+dEQP-GLES2.functional.texture.mipmap.cube.projected.linear_nearest
+dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_linear_clamp
+dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_linear_mirror
+dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_linear_repeat
+dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_nearest_clamp
+dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_nearest_mirror
+dEQP-GLES2.functional.texture.vertex.2d.filtering.linear_mipmap_linear_nearest_repeat
+dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_linear_clamp
+dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_linear_mirror
+dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_linear_repeat
+dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_nearest_clamp
+dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_nearest_mirror
+dEQP-GLES2.functional.texture.vertex.2d.filtering.nearest_mipmap_linear_nearest_repeat
+dEQP-GLES2.functional.texture.vertex.2d.wrap.clamp_clamp
+dEQP-GLES2.functional.texture.vertex.2d.wrap.clamp_mirror
+dEQP-GLES2.functional.texture.vertex.2d.wrap.clamp_repeat
+dEQP-GLES2.functional.texture.vertex.2d.wrap.mirror_clamp
+dEQP-GLES2.functional.texture.vertex.2d.wrap.mirror_mirror
+dEQP-GLES2.functional.texture.vertex.2d.wrap.mirror_repeat
+dEQP-GLES2.functional.texture.vertex.2d.wrap.repeat_clamp
+dEQP-GLES2.functional.texture.vertex.2d.wrap.repeat_mirror
+dEQP-GLES2.functional.texture.vertex.2d.wrap.repeat_repeat
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_clamp
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_mirror
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_linear_repeat
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_clamp
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_mirror
+dEQP-GLES2.functional.texture.vertex.cube.filtering.linear_mipmap_linear_nearest_repeat
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_clamp
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_mirror
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_linear_repeat
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_nearest_clamp
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_nearest_mirror
+dEQP-GLES2.functional.texture.vertex.cube.filtering.nearest_mipmap_linear_nearest_repeat
+dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_clamp
+dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_mirror
+dEQP-GLES2.functional.texture.vertex.cube.wrap.clamp_repeat
+dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_clamp
+dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_mirror
+dEQP-GLES2.functional.texture.vertex.cube.wrap.mirror_repeat
+dEQP-GLES2.functional.texture.vertex.cube.wrap.repeat_clamp
+dEQP-GLES2.functional.texture.vertex.cube.wrap.repeat_mirror
+dEQP-GLES2.functional.texture.vertex.cube.wrap.repeat_repeat

diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh
new file mode 100755 (executable)
index 0000000..34d28a1
--- /dev/null
+++ b/.gitlab-ci/deqp-runner.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+set -ex
+
+DEQP_OPTIONS=(--deqp-surface-width=256 --deqp-surface-height=256)
+DEQP_OPTIONS+=(--deqp-surface-type=pbuffer)
+DEQP_OPTIONS+=(--deqp-gl-config-name=rgba8888d24s8ms0)
+DEQP_OPTIONS+=(--deqp-visibility=hidden)
+DEQP_OPTIONS+=(--deqp-log-images=disable)
+DEQP_OPTIONS+=(--deqp-watchdog=enable)
+DEQP_OPTIONS+=(--deqp-crashhandler=enable)
+
+if [ -z "$DEQP_VER" ]; then
+   echo 'DEQP_VER must be set to something like "gles2" or "gles31" for the test run'
+   exit 1
+fi
+
+if [ -z "$DEQP_SKIPS" ]; then
+   echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"'
+   exit 1
+fi
+
+# Prep the expected failure list
+if [ -n "$DEQP_EXPECTED_FAILS" ]; then
+   export DEQP_EXPECTED_FAILS=`pwd`/artifacts/$DEQP_EXPECTED_FAILS
+else
+   export DEQP_EXPECTED_FAILS=/tmp/expect-no-failures.txt
+   touch $DEQP_EXPECTED_FAILS
+fi
+sort < $DEQP_EXPECTED_FAILS > /tmp/expected-fails.txt
+
+# Fix relative paths on inputs.
+export DEQP_SKIPS=`pwd`/artifacts/$DEQP_SKIPS
+
+# Be a good citizen on the shared runners.
+export LP_NUM_THREADS=4
+
+# Set up the driver environment.
+export LD_LIBRARY_PATH=`pwd`/install/lib/
+export EGL_PLATFORM=surfaceless
+
+# the runner was failing to look for libkms in /usr/local/lib for some reason
+# I never figured out.
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
+RESULTS=`pwd`/results
+mkdir -p $RESULTS
+
+cd /deqp/modules/$DEQP_VER
+
+# Generate test case list file
+cp /deqp/mustpass/$DEQP_VER-master.txt /tmp/case-list.txt
+
+# Note: not using sorted input and comm, becuase I want to run the tests in
+# the same order that dEQP would.
+while read -r line; do
+   if echo "$line" | grep -q '^[^#]'; then
+       sed -i "/$line/d" /tmp/case-list.txt
+   fi
+done < $DEQP_SKIPS
+
+# If the job is parallel, take the corresponding fraction of the caselist.
+# Note: N~M is a gnu sed extension to match every nth line (first line is #1).
+if [ -n "$CI_NODE_INDEX" ]; then
+   sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
+fi
+
+if [ ! -s /tmp/case-list.txt ]; then
+    echo "Caselist generation failed"
+    exit 1
+fi
+
+# Cannot use tee because dash doesn't have pipefail
+touch /tmp/result.txt
+tail -f /tmp/result.txt &
+
+./deqp-$DEQP_VER "${DEQP_OPTIONS[@]}" --deqp-log-filename=$RESULTS/results.qpa --deqp-caselist-file=/tmp/case-list.txt >> /tmp/result.txt
+DEQP_EXITCODE=$?
+
+sed -ne \
+    '/StatusCode="Fail"/{x;p}; s/#beginTestCaseResult //; T; h' \
+    $RESULTS/results.qpa \
+    > /tmp/unsorted-fails.txt
+
+# Scrape out the renderer that the test run used, so we can validate that the
+# right driver was used.
+if grep -q "dEQP-.*.info.renderer" /tmp/case-list.txt; then
+    # This is an ugly dependency on the .qpa format: Print 3 lines after the
+    # match, which happens to contain the result.
+    RENDERER=`sed -n '/#beginTestCaseResult dEQP-.*.info.renderer/{n;n;n;p}' $RESULTS/results.qpa | sed -n -E "s|<Text>(.*)</Text>|\1|p"`
+
+    echo "GL_RENDERER for this test run: $RENDERER"
+
+    if [ -n "$DEQP_RENDERER_MATCH" ]; then
+        echo $RENDERER | grep -q $DEQP_RENDERER_MATCH > /dev/null
+    fi
+fi
+
+if [ $DEQP_EXITCODE -ne 0 ]; then
+   exit $DEQP_EXITCODE
+fi
+
+sort < /tmp/unsorted-fails.txt > $RESULTS/fails.txt
+
+comm -23 $RESULTS/fails.txt /tmp/expected-fails.txt > /tmp/new-fails.txt
+if [ -s /tmp/new-fails.txt ]; then
+    echo "Unexpected failures:"
+    cat /tmp/new-fails.txt
+    exit 1
+else
+    echo "No new failures"
+fi

diff --git a/.gitlab-ci/meson-build.sh b/.gitlab-ci/meson-build.sh
index c2efd2aeedf5f4c36459b3980cd33cc189c77408..7e74095c4bb19e75e0bdc7c8adbc502e82832d5c 100755 (executable)
--- a/.gitlab-ci/meson-build.sh
+++ b/.gitlab-ci/meson-build.sh
@@ -39,3 +39,24 @@ cd ..
 if test -n "$MESON_SHADERDB"; then
     ./.gitlab-ci/run-shader-db.sh;
 fi
+
+# Delete 2MB of includes from artifacts.
+rm -rf install/include
+
+# Strip the drivers in the artifacts to cut 80% of the artifacts size.
+if [ -n "$CROSS" ]; then
+    STRIP=`sed -n -E "s/strip\s*=\s*'(.*)'/\1/p" /cross_file-$CROSS.txt`
+    if [ -z "$STRIP" ]; then
+        echo "Failed to find strip command in cross file"
+        exit 1
+    fi
+else
+    STRIP="strip"
+fi
+find install -name \*.so -exec $STRIP {} \;
+
+# Test runs don't pull down the git tree, so put the dEQP helper
+# script and associated bits there.
+mkdir -p artifacts/
+cp -Rp .gitlab-ci/deqp* artifacts/
+# cp -Rp src/freedreno/ci/expected* artifacts/