From 8a168683d04c51b4df1df8fe0ca72ed8e4fa1c87 Mon Sep 17 00:00:00 2001 From: Tomeu Vizoso Date: Sun, 6 Oct 2019 15:21:39 -0700 Subject: [PATCH] gitlab-ci/lava: Use files to list tests to skip As the non-LAVA runner script does, have per-GPU version files listing the tests that are to be skipped, due to being very slow, unstable, etc. Signed-off-by: Tomeu Vizoso Reviewed-by: Vasily Khoruzhick Reviewed-by: Neil Armstrong --- .gitlab-ci/deqp-panfrost-t760-skips.txt | 63 +++++++++++++++++++++ .gitlab-ci/deqp-panfrost-t860-skips.txt | 63 +++++++++++++++++++++ .gitlab-ci/lava-deqp-runner.sh | 74 +++---------------------- .gitlab-ci/lava-gitlab-ci.yml | 1 + 4 files changed, 134 insertions(+), 67 deletions(-) create mode 100644 .gitlab-ci/deqp-panfrost-t760-skips.txt create mode 100644 .gitlab-ci/deqp-panfrost-t860-skips.txt diff --git a/.gitlab-ci/deqp-panfrost-t760-skips.txt b/.gitlab-ci/deqp-panfrost-t760-skips.txt new file mode 100644 index 00000000000..d387fe80398 --- /dev/null +++ b/.gitlab-ci/deqp-panfrost-t760-skips.txt @@ -0,0 +1,63 @@ +# Note: skips lists for CI are just a list of lines that, when +# non-zero-length and not starting with '#', will regex match to +# delete lines from the test list. Be careful. + +# Skip the perf/stress tests to keep runtime manageable +dEQP-GLES[0-9]*.performance +dEQP-GLES[0-9]*.stress + +# These are really slow on tiling architectures (including llvmpipe). +dEQP-GLES[0-9]*.functional.flush_finish + +dEQP-GLES2.functional.fbo.render.depth.* +dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_neg_x_neg_z_and_pos_x_pos_z_and_neg_x_neg_y_pos_z +dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_pos_y_pos_z_and_neg_x_neg_y_pos_z_and_pos_x_pos_y_neg_z +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgba4 +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgba4 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_stencil_index8 +dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.* +dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.* +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.rbo_rgba4 +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.tex2d_rgb +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.tex2d_rgba +dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4 +dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_rgba4_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.rbo_rgba4_stencil_index8 +dEQP-GLES2.functional.lifetime.attach.deleted_input.renderbuffer_framebuffer +dEQP-GLES2.functional.lifetime.attach.deleted_output.renderbuffer_framebuffer +dEQP-GLES2.functional.polygon_offset.fixed16_factor_0_slope +dEQP-GLES2.functional.polygon_offset.fixed16_factor_1_slope +dEQP-GLES2.functional.shaders.invariance.highp.loop_4 +dEQP-GLES2.functional.shaders.matrix.mul.dynamic_highp_mat4_vec4_vertex +dEQP-GLES2.functional.shaders.matrix.mul.dynamic_highp_vec4_mat4_fragment +dEQP-GLES2.functional.shaders.operator.common_functions.smoothstep.mediump_vec3_vertex +dEQP-GLES2.functional.shaders.random.all_features.fragment.12 +dEQP-GLES2.functional.shaders.random.all_features.fragment.37 +dEQP-GLES2.functional.texture.units.2_units.mixed.1 +dEQP-GLES2.functional.texture.units.2_units.mixed.3 +dEQP-GLES2.functional.texture.units.2_units.only_2d.2 +dEQP-GLES2.functional.texture.units.4_units.mixed.5 +dEQP-GLES2.functional.texture.units.4_units.only_2d.0 +dEQP-GLES2.functional.texture.units.8_units.only_cube.2 +dEQP-GLES2.functional.texture.units.all_units.mixed.6 +dEQP-GLES2.functional.texture.units.all_units.only_cube.4 +dEQP-GLES2.functional.texture.units.all_units.only_cube.7 +dEQP-GLES2.functional.texture.units.all_units.only_cube.8 + diff --git a/.gitlab-ci/deqp-panfrost-t860-skips.txt b/.gitlab-ci/deqp-panfrost-t860-skips.txt new file mode 100644 index 00000000000..d387fe80398 --- /dev/null +++ b/.gitlab-ci/deqp-panfrost-t860-skips.txt @@ -0,0 +1,63 @@ +# Note: skips lists for CI are just a list of lines that, when +# non-zero-length and not starting with '#', will regex match to +# delete lines from the test list. Be careful. + +# Skip the perf/stress tests to keep runtime manageable +dEQP-GLES[0-9]*.performance +dEQP-GLES[0-9]*.stress + +# These are really slow on tiling architectures (including llvmpipe). +dEQP-GLES[0-9]*.functional.flush_finish + +dEQP-GLES2.functional.fbo.render.depth.* +dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_neg_x_neg_z_and_pos_x_pos_z_and_neg_x_neg_y_pos_z +dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_pos_y_pos_z_and_neg_x_neg_y_pos_z_and_pos_x_pos_y_neg_z +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgba4 +dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgba4 +dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_stencil_index8 +dEQP-GLES2.functional.fbo.render.recreate_depthbuffer.* +dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer.* +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.rbo_rgba4 +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.tex2d_rgb +dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.tex2d_rgba +dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgb5_a1 +dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4 +dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgb5_a1_depth_component16 +dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgba4_depth_component16 +dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_rgba4_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.rbo_rgb5_a1_stencil_index8 +dEQP-GLES2.functional.fbo.render.stencil.rbo_rgba4_stencil_index8 +dEQP-GLES2.functional.lifetime.attach.deleted_input.renderbuffer_framebuffer +dEQP-GLES2.functional.lifetime.attach.deleted_output.renderbuffer_framebuffer +dEQP-GLES2.functional.polygon_offset.fixed16_factor_0_slope +dEQP-GLES2.functional.polygon_offset.fixed16_factor_1_slope +dEQP-GLES2.functional.shaders.invariance.highp.loop_4 +dEQP-GLES2.functional.shaders.matrix.mul.dynamic_highp_mat4_vec4_vertex +dEQP-GLES2.functional.shaders.matrix.mul.dynamic_highp_vec4_mat4_fragment +dEQP-GLES2.functional.shaders.operator.common_functions.smoothstep.mediump_vec3_vertex +dEQP-GLES2.functional.shaders.random.all_features.fragment.12 +dEQP-GLES2.functional.shaders.random.all_features.fragment.37 +dEQP-GLES2.functional.texture.units.2_units.mixed.1 +dEQP-GLES2.functional.texture.units.2_units.mixed.3 +dEQP-GLES2.functional.texture.units.2_units.only_2d.2 +dEQP-GLES2.functional.texture.units.4_units.mixed.5 +dEQP-GLES2.functional.texture.units.4_units.only_2d.0 +dEQP-GLES2.functional.texture.units.8_units.only_cube.2 +dEQP-GLES2.functional.texture.units.all_units.mixed.6 +dEQP-GLES2.functional.texture.units.all_units.only_cube.4 +dEQP-GLES2.functional.texture.units.all_units.only_cube.7 +dEQP-GLES2.functional.texture.units.all_units.only_cube.8 + diff --git a/.gitlab-ci/lava-deqp-runner.sh b/.gitlab-ci/lava-deqp-runner.sh index 28302ce375e..3d1eae96f9a 100644 --- a/.gitlab-ci/lava-deqp-runner.sh +++ b/.gitlab-ci/lava-deqp-runner.sh @@ -21,73 +21,13 @@ cd /deqp/modules/gles2 # Generate test case list file ./deqp-gles2 $DEQP_OPTIONS --deqp-runmode=stdout-caselist | grep "TEST: dEQP-GLES2" | cut -d ' ' -f 2 > /tmp/case-list.txt -# Disable for now tests that are very slow, either by just using lots of CPU or by crashing -FLIP_FLOPS=" - dEQP-GLES2.performance - dEQP-GLES2.stress - dEQP-GLES2.functional.fbo.render.depth. - dEQP-GLES2.functional.flush_finish. - " - -# These pass or fail seemingly at random -FLIP_FLOPS="$FLIP_FLOPS - dEQP-GLES2.performance - dEQP-GLES2.stress - dEQP-GLES2.functional.fbo.render.depth. - dEQP-GLES2.functional.flush_finish. - dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_neg_x_neg_z_and_pos_x_pos_z_and_neg_x_neg_y_pos_z - dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_pos_y_pos_z_and_neg_x_neg_y_pos_z_and_pos_x_pos_y_neg_z - dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgb5_a1 - dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgb5_a1_depth_component16 - dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgba4 - dEQP-GLES2.functional.fbo.render.color.blend_rbo_rgba4_depth_component16 - dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgb5_a1 - dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgb5_a1_depth_component16 - dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgba4 - dEQP-GLES2.functional.fbo.render.color.blend_npot_rbo_rgba4_depth_component16 - dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1 - dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1_depth_component16 - dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgb5_a1_stencil_index8 - dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_depth_component16 - dEQP-GLES2.functional.fbo.render.color_clear.rbo_rgba4_stencil_index8 - dEQP-GLES2.functional.fbo.render.recreate_depthbuffer. - dEQP-GLES2.functional.fbo.render.recreate_stencilbuffer. - dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.rbo_rgb5_a1 - dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.rbo_rgba4 - dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.tex2d_rgb - dEQP-GLES2.functional.fbo.render.shared_colorbuffer_clear.tex2d_rgba - dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgb5_a1 - dEQP-GLES2.functional.fbo.render.shared_colorbuffer.rbo_rgba4 - dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgb5_a1_depth_component16 - dEQP-GLES2.functional.fbo.render.shared_depthbuffer.rbo_rgba4_depth_component16 - dEQP-GLES2.functional.fbo.render.stencil_clear.rbo_rgb5_a1_stencil_index8 - dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_rgb5_a1_stencil_index8 - dEQP-GLES2.functional.fbo.render.stencil.npot_rbo_rgba4_stencil_index8 - dEQP-GLES2.functional.fbo.render.stencil.rbo_rgb5_a1_stencil_index8 - dEQP-GLES2.functional.fbo.render.stencil.rbo_rgba4_stencil_index8 - dEQP-GLES2.functional.lifetime.attach.deleted_input.renderbuffer_framebuffer - dEQP-GLES2.functional.lifetime.attach.deleted_output.renderbuffer_framebuffer - dEQP-GLES2.functional.polygon_offset.fixed16_factor_0_slope - dEQP-GLES2.functional.polygon_offset.fixed16_factor_1_slope - dEQP-GLES2.functional.shaders.invariance.highp.loop_4 - dEQP-GLES2.functional.shaders.matrix.mul.dynamic_highp_mat4_vec4_vertex - dEQP-GLES2.functional.shaders.matrix.mul.dynamic_highp_vec4_mat4_fragment - dEQP-GLES2.functional.shaders.operator.common_functions.smoothstep.mediump_vec3_vertex - dEQP-GLES2.functional.shaders.random.all_features.fragment.12 - dEQP-GLES2.functional.shaders.random.all_features.fragment.37 - dEQP-GLES2.functional.texture.units.2_units.mixed.1 - dEQP-GLES2.functional.texture.units.2_units.mixed.3 - dEQP-GLES2.functional.texture.units.2_units.only_2d.2 - dEQP-GLES2.functional.texture.units.4_units.mixed.5 - dEQP-GLES2.functional.texture.units.4_units.only_2d.0 - dEQP-GLES2.functional.texture.units.8_units.only_cube.2 - dEQP-GLES2.functional.texture.units.all_units.mixed.6 - dEQP-GLES2.functional.texture.units.all_units.only_cube.4 - dEQP-GLES2.functional.texture.units.all_units.only_cube.7 - dEQP-GLES2.functional.texture.units.all_units.only_cube.8 - " - -for test in $FLIP_FLOPS; do sed -i "/$test/d" /tmp/case-list.txt; done +# Note: not using sorted input and comm, becuase I want to run the tests in +# the same order that dEQP would. +while read -r line; do + if echo "$line" | grep -q '^[^#]'; then + sed -i "/$line/d" /tmp/case-list.txt + fi +done < /deqp/deqp-$GPU_VERSION-skips.txt /deqp/deqp-volt --cts-build-dir=/deqp \ --threads=8 \ diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml index 1f4f17343c0..432776b81cf 100644 --- a/.gitlab-ci/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava-gitlab-ci.yml @@ -113,6 +113,7 @@ lava-container:arm64: # Pack rootfs - cp .gitlab-ci/lava-deqp-runner.sh /artifacts/rootfs/deqp/. - cp .gitlab-ci/deqp-*-fails.txt /artifacts/rootfs/deqp/. + - cp .gitlab-ci/deqp-*-skips.txt /artifacts/rootfs/deqp/. - du -sh /artifacts/rootfs/deqp/* - find /artifacts/rootfs/ -type f -printf "%s\t%p\n" | sort -n - pushd /artifacts/rootfs/ ; find -H | cpio -H newc -v -o | gzip -c - > $CI_PROJECT_DIR/results/panfrost-rootfs-${DEBIAN_ARCH}.cpio.gz; popd -- 2.30.2