From c2366f01fd194c9aef91153e3050af30c57bb95a Mon Sep 17 00:00:00 2001
From: =?utf8?q?Michel=20D=C3=A4nzer?= <mdaenzer@redhat.com>
Date: Sat, 16 May 2020 17:17:23 +0200
Subject: [PATCH] gitlab-ci: x86_test-base image as common base for
 x86_test-gl/vk

Making use of the relatively recent FDO_BASE_IMAGE feature of the
templates, the x86_test-base image contents are shared as a separate
layer by the x86_test-gl/vk images (meaning the former only needs to be
downloaded once for either or both of the latter). This should be more
efficient in terms of overall network bandwidth and storage, in
particular if the base image changes less often than the -gl/vk ones.

v2:
* List x86_test-base in needs: along with x86_test-gl/vk (see parent
  commit)
* Always put $STABLE/TESTING_EPHEMERAL on separate lines, will make it
  easier to add any non-ephemeral packages

Reviewed-by: Eric Anholt <eric@anholt.net> # v1
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5186>
---
 .gitlab-ci.yml                        | 28 +++++++--
 .gitlab-ci/container/x86_test-base.sh | 69 ++++++++++++++++++++
 .gitlab-ci/container/x86_test-gl.sh   | 90 +++++----------------------
 .gitlab-ci/container/x86_test-vk.sh   | 84 +++++--------------------
 4 files changed, 124 insertions(+), 147 deletions(-)
 create mode 100644 .gitlab-ci/container/x86_test-base.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c3b2c0351aa..6e28b4713db 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,6 +10,7 @@ include:
 
 stages:
   - container
+  - container-2
   - meson-x86_64
   - scons
   - meson-misc
@@ -145,17 +146,34 @@ x86_build:
   needs:
     - x86_build
 
+# Debian 10 based x86 test image base
+x86_test-base:
+  extends: x86_build
+  variables:
+    FDO_DISTRIBUTION_TAG: &x86_test-base "2020-05-23"
+
+.use-x86_test-base:
+  extends:
+    - x86_build
+    - .ci-run-policy
+  stage: container-2
+  variables:
+    BASE_TAG: *x86_test-base
+    FDO_BASE_IMAGE: "$CI_REGISTRY_IMAGE/debian/x86_test-base:$BASE_TAG"
+  needs:
+    - x86_test-base
+
 # Debian 10 based x86 test image for GL
 x86_test-gl:
-  extends: x86_build
+  extends: .use-x86_test-base
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-18"
+    FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-23"
 
 # Debian 10 based x86 test image for VK
 x86_test-vk:
-  extends: x86_build
+  extends: .use-x86_test-base
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-18"
+    FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-23"
 
 # Debian 9 based x86 build image (old LLVM)
 x86_build_old:
@@ -622,6 +640,7 @@ meson-mingw32-x86_64:
   needs:
     - x86_build
     - meson-testing
+    - x86_test-base
     - x86_test-gl
 
 .test-vk:
@@ -633,6 +652,7 @@ meson-mingw32-x86_64:
   needs:
     - x86_build
     - meson-testing
+    - x86_test-base
     - x86_test-vk
 
 .piglit-test:
diff --git a/.gitlab-ci/container/x86_test-base.sh b/.gitlab-ci/container/x86_test-base.sh
new file mode 100644
index 00000000000..e79384537ea
--- /dev/null
+++ b/.gitlab-ci/container/x86_test-base.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get install -y \
+      ca-certificates \
+      gnupg
+
+# Upstream LLVM package repository
+apt-key add .gitlab-ci/container/llvm-snapshot.gpg.key
+echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-9 main" >/etc/apt/sources.list.d/llvm9.list
+
+sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
+echo 'deb https://deb.debian.org/debian testing main' >/etc/apt/sources.list.d/testing.list
+
+apt-get update
+
+# Don't use newer packages from testing by default
+cat >/etc/apt/preferences <<EOF
+Package: *
+Pin: release a=testing
+Pin-Priority: 100
+EOF
+
+apt-get dist-upgrade -y
+
+apt-get install -y --no-remove \
+      git \
+      git-lfs \
+      libexpat1 \
+      libllvm9 \
+      liblz4-1 \
+      libpcre32-3 \
+      libpng16-16 \
+      libpython3.7 \
+      libvulkan1 \
+      libwayland-client0 \
+      libwayland-server0 \
+      libxcb-ewmh2 \
+      libxcb-randr0 \
+      libxcb-keysyms1 \
+      libxcb-xfixes0 \
+      libxkbcommon0 \
+      libxrandr2 \
+      libxrender1 \
+      python \
+      python3-mako \
+      python3-numpy \
+      python3-pil \
+      python3-pytest \
+      python3-requests \
+      python3-six \
+      python3-yaml \
+      python3.7 \
+      qt5-default \
+      qt5-qmake \
+      vulkan-tools \
+      waffle-utils \
+      xauth \
+      xvfb \
+      zlib1g
+
+apt-get purge -y \
+      gnupg
+
+apt-get autoremove -y --purge
diff --git a/.gitlab-ci/container/x86_test-gl.sh b/.gitlab-ci/container/x86_test-gl.sh
index 41816bb99b6..a98e165b8dc 100644
--- a/.gitlab-ci/container/x86_test-gl.sh
+++ b/.gitlab-ci/container/x86_test-gl.sh
@@ -5,83 +5,45 @@ set -o xtrace
 
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get install -y \
-      ca-certificates \
-      gnupg
-
-# Upstream LLVM package repository
-apt-key add .gitlab-ci/container/llvm-snapshot.gpg.key
-echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-9 main" >/etc/apt/sources.list.d/llvm9.list
-
-sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
-echo 'deb https://deb.debian.org/debian testing main' >/etc/apt/sources.list.d/testing.list
-
-apt-get update
-
-# Don't use newer packages from testing by default
-cat >/etc/apt/preferences <<EOF
-Package: *
-Pin: release a=testing
-Pin-Priority: 100
-EOF
-
-apt-get dist-upgrade -y
-
-apt-get install -y --no-remove \
+# Ephemeral packages (installed for this script and removed again at the end)
+STABLE_EPHEMERAL=" \
       autoconf \
       automake \
       ccache \
       cmake \
       g++ \
       gcc \
-      git \
-      libexpat1 \
       libgbm-dev \
       libgles2-mesa-dev \
-      libllvm9 \
       libpcre3-dev \
-      libpcre32-3 \
       libpng-dev \
-      libpng16-16 \
-      libpython3.7 \
       libvulkan-dev \
-      libvulkan1 \
       libwaffle-dev \
-      libwayland-server0 \
-      libxcb-keysyms1 \
       libxcb-keysyms1-dev \
-      libxcb-xfixes0 \
       libxkbcommon-dev \
-      libxkbcommon0 \
       libxrender-dev \
-      libxrender1 \
       make \
       meson \
       patch \
       pkg-config \
-      python \
       python3-distutils \
-      python3-mako \
-      python3-numpy \
-      python3-pil \
-      python3-pytest \
-      python3-requests \
-      python3-six \
-      python3-yaml \
-      python3.7 \
       python3.7-dev \
-      qt5-default \
-      qt5-qmake \
-      waffle-utils \
       wget \
-      xauth \
-      xvfb \
       xz-utils \
-      zlib1g
+      "
+
+TESTING_EPHEMERAL=" \
+      libc6-dev \
+      "
+
+apt-get update
+
+apt-get install -y --no-remove \
+      $STABLE_EPHEMERAL
 
 # Install packages we need from Debian testing last, to avoid pulling in more
 apt-get install -y -t testing \
-      libc6-dev
+      $TESTING_EPHEMERAL
 
 
 . .gitlab-ci/container/container_pre_build.sh
@@ -115,29 +77,7 @@ apt-get install -y -t testing \
 ccache --show-stats
 
 apt-get purge -y \
-      autoconf \
-      automake \
-      ccache \
-      cmake \
-      g++ \
-      gcc \
-      gnupg \
-      libc6-dev \
-      libgbm-dev \
-      libgles2-mesa-dev \
-      libpcre3-dev \
-      libpng-dev \
-      libwaffle-dev \
-      libxcb-keysyms1-dev \
-      libxkbcommon-dev \
-      libxrender-dev \
-      make \
-      meson \
-      patch \
-      pkg-config \
-      python3-distutils \
-      python3.7-dev \
-      wget \
-      xz-utils
+      $STABLE_EPHEMERAL \
+      $TESTING_EPHEMERAL
 
 apt-get autoremove -y --purge
diff --git a/.gitlab-ci/container/x86_test-vk.sh b/.gitlab-ci/container/x86_test-vk.sh
index 0c8983aa09f..670f9e2b0b1 100644
--- a/.gitlab-ci/container/x86_test-vk.sh
+++ b/.gitlab-ci/container/x86_test-vk.sh
@@ -5,74 +5,39 @@ set -o xtrace
 
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get install -y \
-      ca-certificates \
-      gnupg
-
-# Upstream LLVM package repository
-apt-key add .gitlab-ci/container/llvm-snapshot.gpg.key
-echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-9 main" >/etc/apt/sources.list.d/llvm9.list
-
-sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
-echo 'deb https://deb.debian.org/debian testing main' >/etc/apt/sources.list.d/testing.list
-
-apt-get update
-
-# Don't use newer packages from testing by default
-cat >/etc/apt/preferences <<EOF
-Package: *
-Pin: release a=testing
-Pin-Priority: 100
-EOF
-
-apt-get dist-upgrade -y
-
-apt-get install -y --no-remove \
+# Ephemeral packages (installed for this script and removed again at the end)
+STABLE_EPHEMERAL=" \
       ccache \
       cmake \
       g++ \
       gcc \
-      git \
-      git-lfs \
-      libexpat1 \
       libgbm-dev \
       libgles2-mesa-dev \
-      libllvm9 \
-      liblz4-1 \
       liblz4-dev \
       libpng-dev \
-      libpng16-16 \
       libvulkan-dev \
-      libvulkan1 \
-      libwayland-client0 \
-      libwayland-server0 \
       libxcb-ewmh-dev \
-      libxcb-ewmh2 \
-      libxcb-keysyms1 \
       libxcb-keysyms1-dev \
-      libxcb-randr0 \
-      libxcb-xfixes0 \
       libxkbcommon-dev \
-      libxkbcommon0 \
       libxrandr-dev \
-      libxrandr2 \
       libxrender-dev \
-      libxrender1 \
       meson \
       p7zip \
       pkg-config \
-      python \
       python3-distutils \
-      python3-pil \
-      python3-pytest \
-      python3-requests \
-      python3-yaml \
-      vulkan-tools \
       wget \
-      xauth \
-      xvfb
+      "
+
+TESTING_EPHEMERAL=" \
+      libc6-dev \
+      "
+
+apt-get update
 
-# We need multiarch for Wine
+apt-get install -y --no-remove \
+      $STABLE_EPHEMERAL
+
+        # We need multiarch for Wine
 dpkg --add-architecture i386
 
 apt-get update
@@ -84,7 +49,7 @@ apt-get install -y --no-remove \
 
 # Install packages we need from Debian testing last, to avoid pulling in more
 apt-get install -y -t testing \
-      libc6-dev
+      $TESTING_EPHEMERAL
 
 
 ############### Set up Wine env variables
@@ -172,24 +137,7 @@ wine \
 ccache --show-stats
 
 apt-get purge -y \
-      ccache \
-      cmake \
-      g++ \
-      gcc \
-      gnupg \
-      libgbm-dev \
-      libgles2-mesa-dev \
-      liblz4-dev \
-      libpng-dev \
-      libvulkan-dev \
-      libxcb-ewmh-dev \
-      libxcb-keysyms1-dev \
-      libxkbcommon-dev \
-      libxrandr-dev \
-      libxrender-dev \
-      meson \
-      p7zip \
-      pkg-config \
-      wget
+      $STABLE_EPHEMERAL \
+      $TESTING_EPHEMERAL
 
 apt-get autoremove -y --purge
-- 
2.30.2