.gitlab-ci/deqp-runner.sh

   1 #!/bin/bash
   2
   3 set -ex
   4
   5 DEQP_OPTIONS=(--deqp-surface-width=256 --deqp-surface-height=256)
   6 DEQP_OPTIONS+=(--deqp-surface-type=pbuffer)
   7 DEQP_OPTIONS+=(--deqp-gl-config-name=rgba8888d24s8ms0)
   8 DEQP_OPTIONS+=(--deqp-visibility=hidden)
   9 DEQP_OPTIONS+=(--deqp-log-images=disable)
  10 DEQP_OPTIONS+=(--deqp-crashhandler=enable)
  11
  12 # It would be nice to be able to enable the watchdog, so that hangs in a test
  13 # don't need to wait the full hour for the run to time out.  However, some
  14 # shaders end up taking long enough to compile
  15 # (dEQP-GLES31.functional.ubo.random.all_per_block_buffers.20 for example)
  16 # that they'll sporadically trigger the watchdog.
  17 #DEQP_OPTIONS+=(--deqp-watchdog=enable)
  18
  19 if [ -z "$DEQP_VER" ]; then
  20    echo 'DEQP_VER must be set to something like "gles2" or "gles31" for the test run'
  21    exit 1
  22 fi
  23
  24 if [ -z "$DEQP_SKIPS" ]; then
  25    echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"'
  26    exit 1
  27 fi
  28
  29 # Prep the expected failure list
  30 if [ -n "$DEQP_EXPECTED_FAILS" ]; then
  31    export DEQP_EXPECTED_FAILS=`pwd`/artifacts/$DEQP_EXPECTED_FAILS
  32 else
  33    export DEQP_EXPECTED_FAILS=/tmp/expect-no-failures.txt
  34    touch $DEQP_EXPECTED_FAILS
  35 fi
  36 sort < $DEQP_EXPECTED_FAILS > /tmp/expected-fails.txt
  37
  38 # Fix relative paths on inputs.
  39 export DEQP_SKIPS=`pwd`/artifacts/$DEQP_SKIPS
  40
  41 # Be a good citizen on the shared runners.
  42 export LP_NUM_THREADS=4
  43
  44 # Set up the driver environment.
  45 export LD_LIBRARY_PATH=`pwd`/install/lib/
  46 export EGL_PLATFORM=surfaceless
  47
  48 # the runner was failing to look for libkms in /usr/local/lib for some reason
  49 # I never figured out.
  50 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
  51
  52 RESULTS=`pwd`/results
  53 mkdir -p $RESULTS
  54
  55 cd /deqp/modules/$DEQP_VER
  56
  57 # Generate test case list file
  58 cp /deqp/mustpass/$DEQP_VER-master.txt /tmp/case-list.txt
  59
  60 # Note: not using sorted input and comm, becuase I want to run the tests in
  61 # the same order that dEQP would.
  62 while read -r line; do
  63    if echo "$line" | grep -q '^[^#]'; then
  64        sed -i "/$line/d" /tmp/case-list.txt
  65    fi
  66 done < $DEQP_SKIPS
  67
  68 # If the job is parallel, take the corresponding fraction of the caselist.
  69 # Note: N~M is a gnu sed extension to match every nth line (first line is #1).
  70 if [ -n "$CI_NODE_INDEX" ]; then
  71    sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
  72 fi
  73
  74 if [ ! -s /tmp/case-list.txt ]; then
  75     echo "Caselist generation failed"
  76     exit 1
  77 fi
  78
  79 # Cannot use tee because dash doesn't have pipefail
  80 touch /tmp/result.txt
  81 tail -f /tmp/result.txt &
  82
  83 ./deqp-$DEQP_VER "${DEQP_OPTIONS[@]}" --deqp-log-filename=$RESULTS/results.qpa --deqp-caselist-file=/tmp/case-list.txt >> /tmp/result.txt
  84 DEQP_EXITCODE=$?
  85
  86 sed -ne \
  87     '/StatusCode="Fail"/{x;p}; s/#beginTestCaseResult //; T; h' \
  88     $RESULTS/results.qpa \
  89     > /tmp/unsorted-fails.txt
  90
  91 # Scrape out the renderer that the test run used, so we can validate that the
  92 # right driver was used.
  93 if grep -q "dEQP-.*.info.renderer" /tmp/case-list.txt; then
  94     # This is an ugly dependency on the .qpa format: Print 3 lines after the
  95     # match, which happens to contain the result.
  96     RENDERER=`sed -n '/#beginTestCaseResult dEQP-.*.info.renderer/{n;n;n;p}' $RESULTS/results.qpa | sed -n -E "s|<Text>(.*)</Text>|\1|p"`
  97
  98     echo "GL_RENDERER for this test run: $RENDERER"
  99
 100     if [ -n "$DEQP_RENDERER_MATCH" ]; then
 101         echo $RENDERER | grep -q $DEQP_RENDERER_MATCH > /dev/null
 102     fi
 103 fi
 104
 105 if grep -q "dEQP-.*.info.version" /tmp/case-list.txt; then
 106     # This is an ugly dependency on the .qpa format: Print 3 lines after the
 107     # match, which happens to contain the result.
 108     VERSION=`sed -n '/#beginTestCaseResult dEQP-.*.info.version/{n;n;n;p}' $RESULTS/results.qpa | sed -n -E "s|<Text>(.*)</Text>|\1|p"`
 109     echo "Driver version tested: $VERSION"
 110 fi
 111
 112 if [ $DEQP_EXITCODE -ne 0 ]; then
 113    exit $DEQP_EXITCODE
 114 fi
 115
 116 sort < /tmp/unsorted-fails.txt > $RESULTS/fails.txt
 117
 118 comm -23 $RESULTS/fails.txt /tmp/expected-fails.txt > /tmp/new-fails.txt
 119 if [ -s /tmp/new-fails.txt ]; then
 120     echo "Unexpected failures:"
 121     cat /tmp/new-fails.txt
 122     exit 1
 123 else
 124     echo "No new failures"
 125 fi
 126
 127 sort /tmp/case-list.txt > /tmp/sorted-case-list.txt
 128 comm -12 /tmp/sorted-case-list.txt /tmp/expected-fails.txt > /tmp/expected-fails-in-caselist.txt
 129 comm -13 $RESULTS/fails.txt /tmp/expected-fails-in-caselist.txt > /tmp/new-passes.txt
 130 if [ -s /tmp/new-passes.txt ]; then
 131     echo "Unexpected passes, please update $DEQP_EXPECTED_FAILS (or add flaky tests to $DEQP_SKIPS):"
 132     cat /tmp/new-passes.txt
 133     exit 1
 134 else
 135     echo "No new passes"
 136 fi