.gitlab-ci/deqp-runner.sh

   1 #!/bin/bash
   2
   3 set -ex
   4
   5 DEQP_OPTIONS=(--deqp-surface-width=256 --deqp-surface-height=256)
   6 DEQP_OPTIONS+=(--deqp-surface-type=pbuffer)
   7 DEQP_OPTIONS+=(--deqp-gl-config-name=rgba8888d24s8ms0)
   8 DEQP_OPTIONS+=(--deqp-visibility=hidden)
   9
  10 # It would be nice to be able to enable the watchdog, so that hangs in a test
  11 # don't need to wait the full hour for the run to time out.  However, some
  12 # shaders end up taking long enough to compile
  13 # (dEQP-GLES31.functional.ubo.random.all_per_block_buffers.20 for example)
  14 # that they'll sporadically trigger the watchdog.
  15 #DEQP_OPTIONS+=(--deqp-watchdog=enable)
  16
  17 if [ -z "$DEQP_VER" ]; then
  18    echo 'DEQP_VER must be set to something like "gles2" or "gles31" for the test run'
  19    exit 1
  20 fi
  21
  22 if [ -z "$DEQP_SKIPS" ]; then
  23    echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"'
  24    exit 1
  25 fi
  26
  27 ARTIFACTS=`pwd`/artifacts
  28
  29 # Set up the driver environment.
  30 export LD_LIBRARY_PATH=`pwd`/install/lib/
  31 export EGL_PLATFORM=surfaceless
  32
  33 # the runner was failing to look for libkms in /usr/local/lib for some reason
  34 # I never figured out.
  35 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
  36
  37 RESULTS=`pwd`/results
  38 mkdir -p $RESULTS
  39
  40 # Generate test case list file
  41 cp /deqp/mustpass/$DEQP_VER-master.txt /tmp/case-list.txt
  42
  43 # If the job is parallel, take the corresponding fraction of the caselist.
  44 # Note: N~M is a gnu sed extension to match every nth line (first line is #1).
  45 if [ -n "$CI_NODE_INDEX" ]; then
  46    sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
  47 fi
  48
  49 if [ ! -s /tmp/case-list.txt ]; then
  50     echo "Caselist generation failed"
  51     exit 1
  52 fi
  53
  54 if [ -n "$DEQP_EXPECTED_FAILS" ]; then
  55     XFAIL="--xfail-list $ARTIFACTS/$DEQP_EXPECTED_FAILS"
  56 fi
  57
  58 set +e
  59
  60 run_cts() {
  61     caselist=$1
  62     output=$2
  63     deqp-runner \
  64         --deqp /deqp/modules/$DEQP_VER/deqp-$DEQP_VER \
  65         --output $output \
  66         --caselist $caselist \
  67         --exclude-list $ARTIFACTS/$DEQP_SKIPS \
  68         $XFAIL \
  69         --job ${DEQP_PARALLEL:-1} \
  70         --allow-flakes true \
  71         -- \
  72         "${DEQP_OPTIONS[@]}"
  73 }
  74
  75 report_flakes() {
  76     if [ -z "$FLAKES_CHANNEL" ]; then
  77         return 0
  78     fi
  79     flakes=$1
  80     bot="$CI_RUNNER_DESCRIPTION-$CI_PIPELINE_ID"
  81     channel="$FLAKES_CHANNEL"
  82     (
  83     echo NICK $bot
  84     echo USER $bot unused unused :Gitlab CI Notifier
  85     sleep 10
  86     echo "JOIN $channel"
  87     sleep 1
  88     desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION"
  89     if [ -n "CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
  90         desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)"
  91     fi
  92     echo "PRIVMSG $channel :$desc"
  93     for flake in `cat $flakes`; do
  94         echo "PRIVMSG $channel :$flake"
  95     done
  96     echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/"
  97     echo "QUIT"
  98     ) | nc irc.freenode.net 6667 > /dev/null
  99
 100 }
 101
 102 extract_xml_result() {
 103     testcase=$1
 104     shift 1
 105     qpas=$*
 106     start="#beginTestCaseResult $testcase"
 107     for qpa in $qpas; do
 108         while IFS= read -r line; do
 109             if [ "$line" = "$start" ]; then
 110                 dst="$testcase.qpa"
 111                 echo "#beginSession" > $dst
 112                 echo $line >> $dst
 113                 while IFS= read -r line; do
 114                     if [ "$line" = "#endTestCaseResult" ]; then
 115                         echo $line >> $dst
 116                         echo "#endSession" >> $dst
 117                         /deqp/executor/testlog-to-xml $dst "$RESULTS/$testcase.xml"
 118                         # copy the stylesheets here so they only end up in artifacts
 119                         # if we have one or more result xml in artifacts
 120                         cp /deqp/testlog.{css,xsl} "$RESULTS/"
 121                         return 0
 122                     fi
 123                     echo $line >> $dst
 124                 done
 125                 return 1
 126             fi
 127         done < $qpa
 128     done
 129 }
 130
 131 extract_xml_results() {
 132     qpas=$*
 133     while IFS= read -r testcase; do
 134         testcase=${testcase%,*}
 135         extract_xml_result $testcase $qpas
 136     done
 137 }
 138
 139 # Generate junit results
 140 generate_junit() {
 141     results=$1
 142     echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
 143     echo "<testsuites>"
 144     echo "<testsuite name=\"$DEQP_VER-$CI_NODE_INDEX\">"
 145     while read line; do
 146         testcase=${line%,*}
 147         result=${line#*,}
 148         # avoid counting Skip's in the # of tests:
 149         if [ "$result" = "Skip" ]; then
 150             continue;
 151         fi
 152         echo "<testcase name=\"$testcase\">"
 153         if [ "$result" != "Pass" ]; then
 154             echo "<failure type=\"$result\">"
 155             echo "$result: See $CI_JOB_URL/artifacts/results/$testcase.xml"
 156             echo "</failure>"
 157         fi
 158         echo "</testcase>"
 159     done < $results
 160     echo "</testsuite>"
 161     echo "</testsuites>"
 162 }
 163
 164 # wrapper to supress +x to avoid spamming the log
 165 quiet() {
 166     set +x
 167     "$@"
 168     set -x
 169 }
 170
 171 run_cts /tmp/case-list.txt $RESULTS/cts-runner-results.txt
 172 DEQP_EXITCODE=$?
 173
 174 quiet generate_junit $RESULTS/cts-runner-results.txt > $RESULTS/results.xml
 175
 176 if [ $DEQP_EXITCODE -ne 0 ]; then
 177     # preserve caselist files in case of failures:
 178     cp /tmp/cts_runner.*.txt $RESULTS/
 179     echo "Some unexpected results found (see cts-runner-results.txt in artifacts for full results):"
 180     cat $RESULTS/cts-runner-results.txt | \
 181         grep -v ",Pass" | \
 182         grep -v ",Skip" | \
 183         grep -v ",ExpectedFail" > \
 184         $RESULTS/cts-runner-unexpected-results.txt
 185     head -n 50 $RESULTS/cts-runner-unexpected-results.txt
 186
 187     # Save the logs for up to the first 50 unexpected results:
 188     head -n 50 $RESULTS/cts-runner-unexpected-results.txt | quiet extract_xml_results /tmp/*.qpa
 189
 190     count=`cat $RESULTS/cts-runner-unexpected-results.txt | wc -l`
 191
 192     # Re-run fails to detect flakes.  But use a small threshold, if
 193     # something was fundamentally broken, we don't want to re-run
 194     # the entire caselist
 195 else
 196     cat $RESULTS/cts-runner-results.txt | \
 197         grep ",Flake" > \
 198         $RESULTS/cts-runner-flakes.txt
 199
 200     count=`cat $RESULTS/cts-runner-flakes.txt | wc -l`
 201     if [ $count -gt 0 ]; then
 202         echo "Some flakes found (see cts-runner-flakes.txt in artifacts for full results):"
 203         head -n 50 $RESULTS/cts-runner-flakes.txt
 204
 205         # Save the logs for up to the first 50 flakes:
 206         head -n 50 $RESULTS/cts-runner-flakes.txt | quiet extract_xml_results /tmp/*.qpa
 207
 208         # Report the flakes to IRC channel for monitoring (if configured):
 209         quiet report_flakes $RESULTS/cts-runner-flakes.txt
 210     else
 211         # no flakes, so clean-up:
 212         rm $RESULTS/cts-runner-flakes.txt
 213     fi
 214 fi
 215
 216 exit $DEQP_EXITCODE