ci: Use FDO_CI_CONCURRENT as our -j flags when present in the runner env.

fd.o has retuned the x86 runners on packet for -j8.  Rather than having to
tweak our CI every time fd.o decides to rebalance job concurrency, respect
what the runner admin has chosen for their builds (this will also be
convenient for people with large local runners).

Reviewed-by: Michel Dänzer <michel@daenzer.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5669>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d493ea5e34d779185ad769e980874ad605e43fd6..ab224341985f7c6001879e952058715a25c324d0 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -193,7 +193,7 @@ success:
   variables:
     FDO_DISTRIBUTION_VERSION: buster-slim
     FDO_REPO_SUFFIX: "debian/$CI_JOB_NAME"
-    FDO_DISTRIBUTION_EXEC: 'bash .gitlab-ci/container/${CI_JOB_NAME}.sh'
+    FDO_DISTRIBUTION_EXEC: 'env FDO_CI_CONCURRENT=${FDO_CI_CONCURRENT} bash .gitlab-ci/container/${CI_JOB_NAME}.sh'
     # no need to pull the whole repo to build the container image
     GIT_STRATEGY: none
 
@@ -221,7 +221,7 @@ x86_build:
   extends:
     - .use-x86_build-base
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_build "2020-07-10"
+    FDO_DISTRIBUTION_TAG: &x86_build "2020-07-20-jflags"
 
 .use-x86_build:
   variables:
@@ -235,7 +235,7 @@ i386_build:
   extends:
     - .use-x86_build-base
   variables:
-    FDO_DISTRIBUTION_TAG: &i386_build "2020-07-10"
+    FDO_DISTRIBUTION_TAG: &i386_build "2020-07-20-jflags"
 
 .use-i386_build:
   variables:
@@ -249,7 +249,7 @@ ppc64el_build:
   extends:
     - .use-x86_build-base
   variables:
-    FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-10"
+    FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-20-jflags"
 
 .use-ppc64el_build:
   variables:
@@ -263,7 +263,7 @@ s390x_build:
   extends:
     - .use-x86_build-base
   variables:
-    FDO_DISTRIBUTION_TAG: &s390x_build "2020-06-02"
+    FDO_DISTRIBUTION_TAG: &s390x_build "2020-06-29-jflags"
 
 .use-s390x_build:
   variables:
@@ -305,7 +305,7 @@ x86_test-vk:
 x86_build_old:
   extends: x86_build-base
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_build_old "2020-06-12"
+    FDO_DISTRIBUTION_TAG: &x86_build_old "2020-06-29-jflags"
     FDO_DISTRIBUTION_VERSION: stretch-slim
 
 .use-x86_build_old:
@@ -321,7 +321,7 @@ arm_build:
     - .fdo.container-build@debian@arm64v8
     - .container
   variables:
-    FDO_DISTRIBUTION_TAG: &arm_build "2020-07-10"
+    FDO_DISTRIBUTION_TAG: &arm_build "2020-07-20-jflags"
 
 .use-arm_build:
   variables:
@@ -500,10 +500,8 @@ make git archive:
     - .build-linux
     - .use-x86_build
   stage: scons
-  variables:
-    SCONSFLAGS: "-j4"
   script:
-    - .gitlab-ci/scons-build.sh
+    - env SCONSFLAGS="-j${FDO_CI_CONCURRENT:-4}" .gitlab-ci/scons-build.sh
 
 meson-testing:
   extends:
@@ -895,10 +893,9 @@ piglit-quick_shader:
 llvmpipe-gles2:
   variables:
     DEQP_VER: gles2
-    DEQP_PARALLEL: 4
     NIR_VALIDATE: 0
-    # Don't use threads inside llvmpipe, we've already got all 4 cores
-    # busy with DEQP_PARALLEL.
+    # Don't use threads inside llvmpipe, we've already got all cores
+    # busy at the deqp-runner level.
     LP_NUM_THREADS: 0
     DEQP_EXPECTED_FAILS: deqp-llvmpipe-fails.txt
     LIBGL_ALWAYS_SOFTWARE: "true"
@@ -932,11 +929,10 @@ softpipe-gles31:
 virgl-gles2-on-gl:
   variables:
     DEQP_VER: gles2
-    DEQP_PARALLEL: 4
     NIR_VALIDATE: 0
     DEQP_NO_SAVE_RESULTS: 1
-    # Don't use threads inside llvmpipe, we've already got all 4 cores
-    # busy with DEQP_PARALLEL.
+    # Don't use threads inside llvmpipe, we've already got all cores
+    # busy at the deqp-runner level.
     LP_NUM_THREADS: 0
     DEQP_EXPECTED_FAILS: deqp-virgl-gl-fails.txt
     DEQP_OPTIONS: "--deqp-log-images=disable"

diff --git a/.gitlab-ci/container/container_pre_build.sh b/.gitlab-ci/container/container_pre_build.sh
index 97afb0083a48783d8d4d735535f392d28383d1c0..c793e2f6b2c5fb690a1fc049b4a9235326d3a706 100755 (executable)
--- a/.gitlab-ci/container/container_pre_build.sh
+++ b/.gitlab-ci/container/container_pre_build.sh
@@ -14,16 +14,11 @@ export CXX="/usr/lib/ccache/g++"
 
 ccache --show-stats
 
-if uname -m | grep -q arm || uname -m | grep -q aarch64; then
-    export JFLAGS=-j8
-else
-    export JFLAGS=-j4
-fi
-
 # Make a wrapper script for ninja to always include the -j flags
-echo /usr/bin/ninja $JFLAGS '"$@"' > /usr/local/bin/ninja
+echo '#!/bin/sh -x' > /usr/local/bin/ninja
+echo '/usr/bin/ninja -j${FDO_CI_CONCURRENT:-4} "$@"' >> /usr/local/bin/ninja
 chmod +x /usr/local/bin/ninja
 
 # Set MAKEFLAGS so that all make invocations in container builds include the
 # flags (doesn't apply to non-container builds, but we don't run make there)
-export MAKEFLAGS=$JFLAGS
+export MAKEFLAGS="-j${FDO_CI_CONCURRENT:-4}"

diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh
index fb3e87bd55c9665cb9b4324d961923b22e8eb50d..f68dd00600c6d3edce85e230ee98477278a20822 100755 (executable)
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@@ -80,6 +80,10 @@ set +e
 
 if [ -n "$DEQP_PARALLEL" ]; then
    JOB="--job $DEQP_PARALLEL"
+elif [ -n "$FDO_CI_CONCURRENT" ]; then
+   JOB="--job $FDO_CI_CONCURRENT"
+else
+   JOB="--job 4"
 fi
 
 run_cts() {

diff --git a/.gitlab-ci/piglit/run.sh b/.gitlab-ci/piglit/run.sh
index e834cb4e9786f49bc2efa52404d4bb230c0bb9a3..c1f50b9751f5084d6382f11688d1644e8897f749 100755 (executable)
--- a/.gitlab-ci/piglit/run.sh
+++ b/.gitlab-ci/piglit/run.sh
@@ -11,7 +11,7 @@ PIGLIT_OPTIONS=$(echo $PIGLIT_OPTIONS | head -n 1)
 xvfb-run --server-args="-noreset" sh -c \
          "export LD_LIBRARY_PATH=$OLDPWD/install/lib;
          wflinfo --platform glx --api gl --profile core | grep \"Mesa $VERSION\\\$\" &&
-         ./piglit run -j4 $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results"
+         ./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results"
 
 PIGLIT_RESULTS=${PIGLIT_RESULTS:-$PIGLIT_PROFILES}
 mkdir -p .gitlab-ci/piglit