From fd24a959956759dfed987ef83813515b10ffb202 Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Fri, 26 Jun 2020 10:59:41 -0700 Subject: [PATCH] ci: Use FDO_CI_CONCURRENT as our -j flags when present in the runner env. MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit fd.o has retuned the x86 runners on packet for -j8. Rather than having to tweak our CI every time fd.o decides to rebalance job concurrency, respect what the runner admin has chosen for their builds (this will also be convenient for people with large local runners). Reviewed-by: Michel Dänzer Part-of: --- .gitlab-ci.yml | 28 +++++++++------------ .gitlab-ci/container/container_pre_build.sh | 11 +++----- .gitlab-ci/deqp-runner.sh | 4 +++ .gitlab-ci/piglit/run.sh | 2 +- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d493ea5e34d..ab224341985 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,7 +193,7 @@ success: variables: FDO_DISTRIBUTION_VERSION: buster-slim FDO_REPO_SUFFIX: "debian/$CI_JOB_NAME" - FDO_DISTRIBUTION_EXEC: 'bash .gitlab-ci/container/${CI_JOB_NAME}.sh' + FDO_DISTRIBUTION_EXEC: 'env FDO_CI_CONCURRENT=${FDO_CI_CONCURRENT} bash .gitlab-ci/container/${CI_JOB_NAME}.sh' # no need to pull the whole repo to build the container image GIT_STRATEGY: none @@ -221,7 +221,7 @@ x86_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &x86_build "2020-07-10" + FDO_DISTRIBUTION_TAG: &x86_build "2020-07-20-jflags" .use-x86_build: variables: @@ -235,7 +235,7 @@ i386_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &i386_build "2020-07-10" + FDO_DISTRIBUTION_TAG: &i386_build "2020-07-20-jflags" .use-i386_build: variables: @@ -249,7 +249,7 @@ ppc64el_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-10" + FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-20-jflags" .use-ppc64el_build: variables: @@ -263,7 +263,7 @@ s390x_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &s390x_build "2020-06-02" + FDO_DISTRIBUTION_TAG: &s390x_build "2020-06-29-jflags" .use-s390x_build: variables: @@ -305,7 +305,7 @@ x86_test-vk: x86_build_old: extends: x86_build-base variables: - FDO_DISTRIBUTION_TAG: &x86_build_old "2020-06-12" + FDO_DISTRIBUTION_TAG: &x86_build_old "2020-06-29-jflags" FDO_DISTRIBUTION_VERSION: stretch-slim .use-x86_build_old: @@ -321,7 +321,7 @@ arm_build: - .fdo.container-build@debian@arm64v8 - .container variables: - FDO_DISTRIBUTION_TAG: &arm_build "2020-07-10" + FDO_DISTRIBUTION_TAG: &arm_build "2020-07-20-jflags" .use-arm_build: variables: @@ -500,10 +500,8 @@ make git archive: - .build-linux - .use-x86_build stage: scons - variables: - SCONSFLAGS: "-j4" script: - - .gitlab-ci/scons-build.sh + - env SCONSFLAGS="-j${FDO_CI_CONCURRENT:-4}" .gitlab-ci/scons-build.sh meson-testing: extends: @@ -895,10 +893,9 @@ piglit-quick_shader: llvmpipe-gles2: variables: DEQP_VER: gles2 - DEQP_PARALLEL: 4 NIR_VALIDATE: 0 - # Don't use threads inside llvmpipe, we've already got all 4 cores - # busy with DEQP_PARALLEL. + # Don't use threads inside llvmpipe, we've already got all cores + # busy at the deqp-runner level. LP_NUM_THREADS: 0 DEQP_EXPECTED_FAILS: deqp-llvmpipe-fails.txt LIBGL_ALWAYS_SOFTWARE: "true" @@ -932,11 +929,10 @@ softpipe-gles31: virgl-gles2-on-gl: variables: DEQP_VER: gles2 - DEQP_PARALLEL: 4 NIR_VALIDATE: 0 DEQP_NO_SAVE_RESULTS: 1 - # Don't use threads inside llvmpipe, we've already got all 4 cores - # busy with DEQP_PARALLEL. + # Don't use threads inside llvmpipe, we've already got all cores + # busy at the deqp-runner level. LP_NUM_THREADS: 0 DEQP_EXPECTED_FAILS: deqp-virgl-gl-fails.txt DEQP_OPTIONS: "--deqp-log-images=disable" diff --git a/.gitlab-ci/container/container_pre_build.sh b/.gitlab-ci/container/container_pre_build.sh index 97afb0083a4..c793e2f6b2c 100755 --- a/.gitlab-ci/container/container_pre_build.sh +++ b/.gitlab-ci/container/container_pre_build.sh @@ -14,16 +14,11 @@ export CXX="/usr/lib/ccache/g++" ccache --show-stats -if uname -m | grep -q arm || uname -m | grep -q aarch64; then - export JFLAGS=-j8 -else - export JFLAGS=-j4 -fi - # Make a wrapper script for ninja to always include the -j flags -echo /usr/bin/ninja $JFLAGS '"$@"' > /usr/local/bin/ninja +echo '#!/bin/sh -x' > /usr/local/bin/ninja +echo '/usr/bin/ninja -j${FDO_CI_CONCURRENT:-4} "$@"' >> /usr/local/bin/ninja chmod +x /usr/local/bin/ninja # Set MAKEFLAGS so that all make invocations in container builds include the # flags (doesn't apply to non-container builds, but we don't run make there) -export MAKEFLAGS=$JFLAGS +export MAKEFLAGS="-j${FDO_CI_CONCURRENT:-4}" diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh index fb3e87bd55c..f68dd00600c 100755 --- a/.gitlab-ci/deqp-runner.sh +++ b/.gitlab-ci/deqp-runner.sh @@ -80,6 +80,10 @@ set +e if [ -n "$DEQP_PARALLEL" ]; then JOB="--job $DEQP_PARALLEL" +elif [ -n "$FDO_CI_CONCURRENT" ]; then + JOB="--job $FDO_CI_CONCURRENT" +else + JOB="--job 4" fi run_cts() { diff --git a/.gitlab-ci/piglit/run.sh b/.gitlab-ci/piglit/run.sh index e834cb4e978..c1f50b9751f 100755 --- a/.gitlab-ci/piglit/run.sh +++ b/.gitlab-ci/piglit/run.sh @@ -11,7 +11,7 @@ PIGLIT_OPTIONS=$(echo $PIGLIT_OPTIONS | head -n 1) xvfb-run --server-args="-noreset" sh -c \ "export LD_LIBRARY_PATH=$OLDPWD/install/lib; wflinfo --platform glx --api gl --profile core | grep \"Mesa $VERSION\\\$\" && - ./piglit run -j4 $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results" + ./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results" PIGLIT_RESULTS=${PIGLIT_RESULTS:-$PIGLIT_PROFILES} mkdir -p .gitlab-ci/piglit -- 2.30.2