gitlab-ci: x86_test-base image as common base for x86_test-gl/vk

Making use of the relatively recent FDO_BASE_IMAGE feature of the
templates, the x86_test-base image contents are shared as a separate
layer by the x86_test-gl/vk images (meaning the former only needs to be
downloaded once for either or both of the latter). This should be more
efficient in terms of overall network bandwidth and storage, in
particular if the base image changes less often than the -gl/vk ones.

v2:
* List x86_test-base in needs: along with x86_test-gl/vk (see parent
  commit)
* Always put $STABLE/TESTING_EPHEMERAL on separate lines, will make it
  easier to add any non-ephemeral packages

Reviewed-by: Eric Anholt <eric@anholt.net> # v1
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5186>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c3b2c0351aab52db36bc230e5743581d968f8aa5..6e28b4713db6f205f101624b73ba32a06afb6613 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,6 +10,7 @@ include:
 
 stages:
   - container
+  - container-2
   - meson-x86_64
   - scons
   - meson-misc
@@ -145,17 +146,34 @@ x86_build:
   needs:
     - x86_build
 
+# Debian 10 based x86 test image base
+x86_test-base:
+  extends: x86_build
+  variables:
+    FDO_DISTRIBUTION_TAG: &x86_test-base "2020-05-23"
+
+.use-x86_test-base:
+  extends:
+    - x86_build
+    - .ci-run-policy
+  stage: container-2
+  variables:
+    BASE_TAG: *x86_test-base
+    FDO_BASE_IMAGE: "$CI_REGISTRY_IMAGE/debian/x86_test-base:$BASE_TAG"
+  needs:
+    - x86_test-base
+
 # Debian 10 based x86 test image for GL
 x86_test-gl:
-  extends: x86_build
+  extends: .use-x86_test-base
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-18"
+    FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-23"
 
 # Debian 10 based x86 test image for VK
 x86_test-vk:
-  extends: x86_build
+  extends: .use-x86_test-base
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-18"
+    FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-23"
 
 # Debian 9 based x86 build image (old LLVM)
 x86_build_old:
@@ -622,6 +640,7 @@ meson-mingw32-x86_64:
   needs:
     - x86_build
     - meson-testing
+    - x86_test-base
     - x86_test-gl
 
 .test-vk:
@@ -633,6 +652,7 @@ meson-mingw32-x86_64:
   needs:
     - x86_build
     - meson-testing
+    - x86_test-base
     - x86_test-vk
 
 .piglit-test:

diff --git a/.gitlab-ci/container/x86_test-base.sh b/.gitlab-ci/container/x86_test-base.sh
new file mode 100644 (file)
index 0000000..e793845
--- /dev/null
+++ b/.gitlab-ci/container/x86_test-base.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get install -y \
+      ca-certificates \
+      gnupg
+
+# Upstream LLVM package repository
+apt-key add .gitlab-ci/container/llvm-snapshot.gpg.key
+echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-9 main" >/etc/apt/sources.list.d/llvm9.list
+
+sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
+echo 'deb https://deb.debian.org/debian testing main' >/etc/apt/sources.list.d/testing.list
+
+apt-get update
+
+# Don't use newer packages from testing by default
+cat >/etc/apt/preferences <<EOF
+Package: *
+Pin: release a=testing
+Pin-Priority: 100
+EOF
+
+apt-get dist-upgrade -y
+
+apt-get install -y --no-remove \
+      git \
+      git-lfs \
+      libexpat1 \
+      libllvm9 \
+      liblz4-1 \
+      libpcre32-3 \
+      libpng16-16 \
+      libpython3.7 \
+      libvulkan1 \
+      libwayland-client0 \
+      libwayland-server0 \
+      libxcb-ewmh2 \
+      libxcb-randr0 \
+      libxcb-keysyms1 \
+      libxcb-xfixes0 \
+      libxkbcommon0 \
+      libxrandr2 \
+      libxrender1 \
+      python \
+      python3-mako \
+      python3-numpy \
+      python3-pil \
+      python3-pytest \
+      python3-requests \
+      python3-six \
+      python3-yaml \
+      python3.7 \
+      qt5-default \
+      qt5-qmake \
+      vulkan-tools \
+      waffle-utils \
+      xauth \
+      xvfb \
+      zlib1g
+
+apt-get purge -y \
+      gnupg
+
+apt-get autoremove -y --purge

diff --git a/.gitlab-ci/container/x86_test-gl.sh b/.gitlab-ci/container/x86_test-gl.sh
index 41816bb99b6c2cfd8a4c5b7be3d4bcb6edf465fc..a98e165b8dc657ffe400e6d45161059212a13ee9 100644 (file)
--- a/.gitlab-ci/container/x86_test-gl.sh
+++ b/.gitlab-ci/container/x86_test-gl.sh
@@ -5,83 +5,45 @@ set -o xtrace
 
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get install -y \
-      ca-certificates \
-      gnupg
-
-# Upstream LLVM package repository
-apt-key add .gitlab-ci/container/llvm-snapshot.gpg.key
-echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-9 main" >/etc/apt/sources.list.d/llvm9.list
-
-sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
-echo 'deb https://deb.debian.org/debian testing main' >/etc/apt/sources.list.d/testing.list
-
-apt-get update
-
-# Don't use newer packages from testing by default
-cat >/etc/apt/preferences <<EOF
-Package: *
-Pin: release a=testing
-Pin-Priority: 100
-EOF
-
-apt-get dist-upgrade -y
-
-apt-get install -y --no-remove \
+# Ephemeral packages (installed for this script and removed again at the end)
+STABLE_EPHEMERAL=" \
       autoconf \
       automake \
       ccache \
       cmake \
       g++ \
       gcc \
-      git \
-      libexpat1 \
       libgbm-dev \
       libgles2-mesa-dev \
-      libllvm9 \
       libpcre3-dev \
-      libpcre32-3 \
       libpng-dev \
-      libpng16-16 \
-      libpython3.7 \
       libvulkan-dev \
-      libvulkan1 \
       libwaffle-dev \
-      libwayland-server0 \
-      libxcb-keysyms1 \
       libxcb-keysyms1-dev \
-      libxcb-xfixes0 \
       libxkbcommon-dev \
-      libxkbcommon0 \
       libxrender-dev \
-      libxrender1 \
       make \
       meson \
       patch \
       pkg-config \
-      python \
       python3-distutils \
-      python3-mako \
-      python3-numpy \
-      python3-pil \
-      python3-pytest \
-      python3-requests \
-      python3-six \
-      python3-yaml \
-      python3.7 \
       python3.7-dev \
-      qt5-default \
-      qt5-qmake \
-      waffle-utils \
       wget \
-      xauth \
-      xvfb \
       xz-utils \
-      zlib1g
+      "
+
+TESTING_EPHEMERAL=" \
+      libc6-dev \
+      "
+
+apt-get update
+
+apt-get install -y --no-remove \
+      $STABLE_EPHEMERAL
 
 # Install packages we need from Debian testing last, to avoid pulling in more
 apt-get install -y -t testing \
-      libc6-dev
+      $TESTING_EPHEMERAL
 
 
 . .gitlab-ci/container/container_pre_build.sh
@@ -115,29 +77,7 @@ apt-get install -y -t testing \
 ccache --show-stats
 
 apt-get purge -y \
-      autoconf \
-      automake \
-      ccache \
-      cmake \
-      g++ \
-      gcc \
-      gnupg \
-      libc6-dev \
-      libgbm-dev \
-      libgles2-mesa-dev \
-      libpcre3-dev \
-      libpng-dev \
-      libwaffle-dev \
-      libxcb-keysyms1-dev \
-      libxkbcommon-dev \
-      libxrender-dev \
-      make \
-      meson \
-      patch \
-      pkg-config \
-      python3-distutils \
-      python3.7-dev \
-      wget \
-      xz-utils
+      $STABLE_EPHEMERAL \
+      $TESTING_EPHEMERAL
 
 apt-get autoremove -y --purge

diff --git a/.gitlab-ci/container/x86_test-vk.sh b/.gitlab-ci/container/x86_test-vk.sh
index 0c8983aa09fc3964b8b88247b7a36fb64a4a0b27..670f9e2b0b1e4ec7425d4371e34f734a859e333c 100644 (file)
--- a/.gitlab-ci/container/x86_test-vk.sh
+++ b/.gitlab-ci/container/x86_test-vk.sh
@@ -5,74 +5,39 @@ set -o xtrace
 
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get install -y \
-      ca-certificates \
-      gnupg
-
-# Upstream LLVM package repository
-apt-key add .gitlab-ci/container/llvm-snapshot.gpg.key
-echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-9 main" >/etc/apt/sources.list.d/llvm9.list
-
-sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
-echo 'deb https://deb.debian.org/debian testing main' >/etc/apt/sources.list.d/testing.list
-
-apt-get update
-
-# Don't use newer packages from testing by default
-cat >/etc/apt/preferences <<EOF
-Package: *
-Pin: release a=testing
-Pin-Priority: 100
-EOF
-
-apt-get dist-upgrade -y
-
-apt-get install -y --no-remove \
+# Ephemeral packages (installed for this script and removed again at the end)
+STABLE_EPHEMERAL=" \
       ccache \
       cmake \
       g++ \
       gcc \
-      git \
-      git-lfs \
-      libexpat1 \
       libgbm-dev \
       libgles2-mesa-dev \
-      libllvm9 \
-      liblz4-1 \
       liblz4-dev \
       libpng-dev \
-      libpng16-16 \
       libvulkan-dev \
-      libvulkan1 \
-      libwayland-client0 \
-      libwayland-server0 \
       libxcb-ewmh-dev \
-      libxcb-ewmh2 \
-      libxcb-keysyms1 \
       libxcb-keysyms1-dev \
-      libxcb-randr0 \
-      libxcb-xfixes0 \
       libxkbcommon-dev \
-      libxkbcommon0 \
       libxrandr-dev \
-      libxrandr2 \
       libxrender-dev \
-      libxrender1 \
       meson \
       p7zip \
       pkg-config \
-      python \
       python3-distutils \
-      python3-pil \
-      python3-pytest \
-      python3-requests \
-      python3-yaml \
-      vulkan-tools \
       wget \
-      xauth \
-      xvfb
+      "
+
+TESTING_EPHEMERAL=" \
+      libc6-dev \
+      "
+
+apt-get update
 
-# We need multiarch for Wine
+apt-get install -y --no-remove \
+      $STABLE_EPHEMERAL
+
+        # We need multiarch for Wine
 dpkg --add-architecture i386
 
 apt-get update
@@ -84,7 +49,7 @@ apt-get install -y --no-remove \
 
 # Install packages we need from Debian testing last, to avoid pulling in more
 apt-get install -y -t testing \
-      libc6-dev
+      $TESTING_EPHEMERAL
 
 
 ############### Set up Wine env variables
@@ -172,24 +137,7 @@ wine \
 ccache --show-stats
 
 apt-get purge -y \
-      ccache \
-      cmake \
-      g++ \
-      gcc \
-      gnupg \
-      libgbm-dev \
-      libgles2-mesa-dev \
-      liblz4-dev \
-      libpng-dev \
-      libvulkan-dev \
-      libxcb-ewmh-dev \
-      libxcb-keysyms1-dev \
-      libxkbcommon-dev \
-      libxrandr-dev \
-      libxrender-dev \
-      meson \
-      p7zip \
-      pkg-config \
-      wget
+      $STABLE_EPHEMERAL \
+      $TESTING_EPHEMERAL
 
 apt-get autoremove -y --purge