panfrost: ci: Switch from direct Docker use to buildah
authorTomeu Vizoso <tomeu.vizoso@collabora.com>
Fri, 7 Jun 2019 08:20:28 +0000 (10:20 +0200)
committerTomeu Vizoso <tomeu.vizoso@collabora.com>
Mon, 10 Jun 2019 06:09:23 +0000 (08:09 +0200)
Use the infrastructure in wayland/ci-templates to build the container
images.

This prevents from getting into some situations in which the images
wouldn't be rebuilt, and allows us to share some infrastructure with
other projects in freedesktop.org.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Suggested-by: Michel Dänzer <michel@daenzer.net>
Acked-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
src/gallium/drivers/panfrost/ci/Dockerfile [deleted file]
src/gallium/drivers/panfrost/ci/debian-install.sh [new file with mode: 0644]
src/gallium/drivers/panfrost/ci/gitlab-ci.yml

diff --git a/src/gallium/drivers/panfrost/ci/Dockerfile b/src/gallium/drivers/panfrost/ci/Dockerfile
deleted file mode 100644 (file)
index fe0e67c..0000000
+++ /dev/null
@@ -1,135 +0,0 @@
-FROM debian:testing
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
-RUN echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
-RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d
-RUN echo 'exit 101' >> /usr/sbin/policy-rc.d
-RUN chmod +x /usr/sbin/policy-rc.d
-
-############### Install packages for building
-
-ARG DEBIAN_ARCH
-RUN dpkg --add-architecture ${DEBIAN_ARCH}
-RUN echo deb-src http://deb.debian.org/debian testing main >> /etc/apt/sources.list
-RUN apt-get update && \
-    apt-get -y install ca-certificates && \
-    apt-get -y install --no-install-recommends \
-      crossbuild-essential-${DEBIAN_ARCH} \
-      meson \
-      g++ \
-      git \
-      ccache \
-      pkg-config \
-      python3-mako \
-      python-numpy \
-      python-six \
-      python-mako \
-      python3-pip \
-      python3-setuptools \
-      python3-six \
-      python3-wheel \
-      python3-jinja2 \
-      bison \
-      flex \
-      libwayland-dev \
-      gettext \
-      cmake \
-      bc \
-      libssl-dev \
-      lavacli \
-      csvkit \
-      curl \
-      unzip \
-      wget \
-      debootstrap \
-      procps \
-      qemu-user-static \
-      cpio \
-      \
-      libdrm-dev:${DEBIAN_ARCH} \
-      libx11-dev:${DEBIAN_ARCH} \
-      libxxf86vm-dev:${DEBIAN_ARCH} \
-      libexpat1-dev:${DEBIAN_ARCH} \
-      libsensors-dev:${DEBIAN_ARCH} \
-      libxfixes-dev:${DEBIAN_ARCH} \
-      libxdamage-dev:${DEBIAN_ARCH} \
-      libxext-dev:${DEBIAN_ARCH} \
-      x11proto-dev:${DEBIAN_ARCH} \
-      libx11-xcb-dev:${DEBIAN_ARCH} \
-      libxcb-dri2-0-dev:${DEBIAN_ARCH} \
-      libxcb-glx0-dev:${DEBIAN_ARCH} \
-      libxcb-xfixes0-dev:${DEBIAN_ARCH} \
-      libxcb-dri3-dev:${DEBIAN_ARCH} \
-      libxcb-present-dev:${DEBIAN_ARCH} \
-      libxcb-randr0-dev:${DEBIAN_ARCH} \
-      libxcb-sync-dev:${DEBIAN_ARCH} \
-      libxrandr-dev:${DEBIAN_ARCH} \
-      libxshmfence-dev:${DEBIAN_ARCH} \
-      libelf-dev:${DEBIAN_ARCH} \
-      libwayland-dev:${DEBIAN_ARCH} \
-      libwayland-egl-backend-dev:${DEBIAN_ARCH} \
-      libclang-7-dev:${DEBIAN_ARCH} \
-      zlib1g-dev:${DEBIAN_ARCH} \
-      libglvnd-core-dev:${DEBIAN_ARCH} \
-      wayland-protocols:${DEBIAN_ARCH} \
-      libpng-dev:${DEBIAN_ARCH} && \
-    rm -rf /var/lib/apt/lists
-
-############### Cross-build dEQP
-ARG GCC_ARCH
-RUN mkdir -p /artifacts/rootfs/deqp                                             && \
-  wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip && \
-  unzip -q opengl-es-cts-3.2.5.0.zip -d /                                       && \
-  rm opengl-es-cts-3.2.5.0.zip                                                  && \
-  cd /VK-GL-CTS-opengl-es-cts-3.2.5.0                                           && \
-  python3 external/fetch_sources.py                                             && \
-  cd /artifacts/rootfs/deqp                                                     && \
-  cmake -DDEQP_TARGET=wayland                                                      \
-    -DCMAKE_BUILD_TYPE=Release                                                     \
-    -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc                                             \
-    -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++                                           \
-    /VK-GL-CTS-opengl-es-cts-3.2.5.0                                            && \
-  make -j$(nproc)                                                               && \
-  rm -rf /artifacts/rootfs/deqp/external                                        && \
-  rm -rf /artifacts/rootfs/deqp/modules/gles3                                   && \
-  rm -rf /artifacts/rootfs/deqp/modules/gles31                                  && \
-  rm -rf /artifacts/rootfs/deqp/modules/internal                                && \
-  rm -rf /artifacts/rootfs/deqp/executor                                        && \
-  rm -rf /artifacts/rootfs/deqp/execserver                                      && \
-  rm -rf /artifacts/rootfs/deqp/modules/egl                                     && \
-  rm -rf /artifacts/rootfs/deqp/framework                                       && \
-  find . -name CMakeFiles | xargs rm -rf                                        && \
-  find . -name lib\*.a | xargs rm -rf                                           && \
-  du -sh *                                                                      && \
-  rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
-
-
-############### Cross-build kernel
-
-ARG KERNEL_ARCH
-ARG DEFCONFIG
-ARG DEVICE_TREES
-ARG KERNEL_IMAGE_NAME
-ENV KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz"
-
-COPY ${KERNEL_ARCH}.config /panfrost-ci/
-RUN mkdir -p /kernel                                                                   && \
-  wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel                                        && \
-  cd /kernel                                                                           && \
-  ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" ./scripts/kconfig/merge_config.sh ${DEFCONFIG} /panfrost-ci/${KERNEL_ARCH}.config && \
-  ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" make -j12 ${KERNEL_IMAGE_NAME} dtbs && \
-  cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/.                        && \
-  cp ${DEVICE_TREES} /artifacts/.                                                      && \
-  rm -rf /kernel
-
-
-############### Create rootfs
-
-COPY create-rootfs.sh /artifacts/rootfs/
-RUN debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian && \
-    chroot /artifacts/rootfs sh /create-rootfs.sh                                                               && \
-    rm /artifacts/rootfs/create-rootfs.sh
-
-ENTRYPOINT [""]
diff --git a/src/gallium/drivers/panfrost/ci/debian-install.sh b/src/gallium/drivers/panfrost/ci/debian-install.sh
new file mode 100644 (file)
index 0000000..8a8b4ec
--- /dev/null
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+PANFROST_CI_DIR=/tmp/clone/src/gallium/drivers/panfrost/ci
+
+############### Install packages for building
+dpkg --add-architecture ${DEBIAN_ARCH}
+echo 'deb-src https://deb.debian.org/debian testing main' > /etc/apt/sources.list.d/deb-src.list
+apt-get update
+apt-get -y install ca-certificates
+apt-get -y install --no-install-recommends \
+       crossbuild-essential-${DEBIAN_ARCH} \
+       meson \
+       g++ \
+       git \
+       ccache \
+       pkg-config \
+       python3-mako \
+       python-numpy \
+       python-six \
+       python-mako \
+       python3-pip \
+       python3-setuptools \
+       python3-six \
+       python3-wheel \
+       python3-jinja2 \
+       bison \
+       flex \
+       libwayland-dev \
+       gettext \
+       cmake \
+       bc \
+       libssl-dev \
+       lavacli \
+       csvkit \
+       curl \
+       unzip \
+       wget \
+       debootstrap \
+       procps \
+       qemu-user-static \
+       cpio \
+       \
+       libdrm-dev:${DEBIAN_ARCH} \
+       libx11-dev:${DEBIAN_ARCH} \
+       libxxf86vm-dev:${DEBIAN_ARCH} \
+       libexpat1-dev:${DEBIAN_ARCH} \
+       libsensors-dev:${DEBIAN_ARCH} \
+       libxfixes-dev:${DEBIAN_ARCH} \
+       libxdamage-dev:${DEBIAN_ARCH} \
+       libxext-dev:${DEBIAN_ARCH} \
+       x11proto-dev:${DEBIAN_ARCH} \
+       libx11-xcb-dev:${DEBIAN_ARCH} \
+       libxcb-dri2-0-dev:${DEBIAN_ARCH} \
+       libxcb-glx0-dev:${DEBIAN_ARCH} \
+       libxcb-xfixes0-dev:${DEBIAN_ARCH} \
+       libxcb-dri3-dev:${DEBIAN_ARCH} \
+       libxcb-present-dev:${DEBIAN_ARCH} \
+       libxcb-randr0-dev:${DEBIAN_ARCH} \
+       libxcb-sync-dev:${DEBIAN_ARCH} \
+       libxrandr-dev:${DEBIAN_ARCH} \
+       libxshmfence-dev:${DEBIAN_ARCH} \
+       libelf-dev:${DEBIAN_ARCH} \
+       libwayland-dev:${DEBIAN_ARCH} \
+       libwayland-egl-backend-dev:${DEBIAN_ARCH} \
+       libclang-7-dev:${DEBIAN_ARCH} \
+       zlib1g-dev:${DEBIAN_ARCH} \
+       libglvnd-core-dev:${DEBIAN_ARCH} \
+       wayland-protocols:${DEBIAN_ARCH} \
+       libpng-dev:${DEBIAN_ARCH}
+
+############### Cross-build dEQP
+mkdir -p /artifacts/rootfs/deqp
+
+wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip
+unzip -q opengl-es-cts-3.2.5.0.zip -d /
+rm opengl-es-cts-3.2.5.0.zip
+
+cd /VK-GL-CTS-opengl-es-cts-3.2.5.0
+python3 external/fetch_sources.py
+
+cd /artifacts/rootfs/deqp
+cmake -DDEQP_TARGET=wayland                   \
+      -DCMAKE_BUILD_TYPE=Release              \
+      -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc      \
+      -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++    \
+      /VK-GL-CTS-opengl-es-cts-3.2.5.0
+make -j$(nproc)
+rm -rf /artifacts/rootfs/deqp/external
+rm -rf /artifacts/rootfs/deqp/modules/gles3
+rm -rf /artifacts/rootfs/deqp/modules/gles31
+rm -rf /artifacts/rootfs/deqp/modules/internal
+rm -rf /artifacts/rootfs/deqp/executor
+rm -rf /artifacts/rootfs/deqp/execserver
+rm -rf /artifacts/rootfs/deqp/modules/egl
+rm -rf /artifacts/rootfs/deqp/framework
+find . -name CMakeFiles | xargs rm -rf
+find . -name lib\*.a | xargs rm -rf
+du -sh *
+rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0
+
+
+############### Cross-build kernel
+KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz"
+export ARCH=${KERNEL_ARCH}
+export CROSS_COMPILE="${GCC_ARCH}-"
+
+mkdir -p /kernel
+wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel
+cd /kernel
+./scripts/kconfig/merge_config.sh ${DEFCONFIG} ${PANFROST_CI_DIR}/${KERNEL_ARCH}.config
+make -j12 ${KERNEL_IMAGE_NAME} dtbs
+cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/.
+cp ${DEVICE_TREES} /artifacts/.
+rm -rf /kernel
+
+
+############### Create rootfs
+cp ${PANFROST_CI_DIR}/create-rootfs.sh /artifacts/rootfs/.
+debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian
+chroot /artifacts/rootfs sh /create-rootfs.sh
+rm /artifacts/rootfs/create-rootfs.sh
+
index 3491b9f98752bab991451097efcf17da5a8982ed..d67c1ff6c9464fc046f64e6b9e9192e2c41a00d7 100644 (file)
@@ -1,39 +1,55 @@
+# DEBIAN_TAG is the tag of the docker image used for the build jobs. If the
+# image doesn't exist yet, the docker-image stage generates it.
+#
+# In order to generate a new image, one should generally change the tag.
+# While removing the image from the registry would also work, that's not
+# recommended except for ephemeral images during development: Replacing an
+# image after a significant amount of time might pull in newer versions of
+# gcc/clang or other packages, which might break the build with older commits
+# using the same tag.
+#
+# After merging a change resulting in generating a new image to the main
+# repository, it's recommended to remove the image from the source repository's
+# container registry, so that the image from the main repository's registry
+# will be used there as well.  You can manage your images on your fork of:
+# https://gitlab.freedesktop.org/xorg/xserver/container_registry
 variables:
-  DOCKER_DRIVER: overlay2
+  UPSTREAM_REPO: mesa/mesa
+  DEBIAN_VERSION: testing-slim
+  DEBIAN_TAG: "2019-06-07-1"
+
+include:
+  - project: 'wayland/ci-templates'
+    ref: c73dae8b84697ef18e2dbbf4fed7386d9652b0cd
+    file: '/templates/debian.yml'
 
 stages:
   - containers
   - build
   - test
 
+# Retry jobs after runner system failures
+.retry: &retry
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+
 # Build Docker image with deqp, the rootfs and the build deps for Mesa
 .container:
+  extends: .debian@container-ifnot-exists
   stage: containers
-  image: docker:stable
-  only:
-    changes:
-      - src/gallium/drivers/panfrost/ci/gitlab-ci.yml
-      - src/gallium/drivers/panfrost/ci/Dockerfile
-      - src/gallium/drivers/panfrost/ci/create-rootfs.sh
-      - src/gallium/drivers/panfrost/ci/${KERNEL_ARCH}.config
-  services:
-    - docker:dind
+  <<: *retry
   variables:
-    DOCKER_HOST: tcp://docker:2375
-    DOCKER_DRIVER: overlay2
-  script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - time docker build
-             -t $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
-             --build-arg DEBIAN_ARCH=${DEBIAN_ARCH}
-             --build-arg GCC_ARCH=${GCC_ARCH}
-             --build-arg KERNEL_ARCH=${KERNEL_ARCH}
-             --build-arg DEFCONFIG=${DEFCONFIG}
-             --build-arg DEVICE_TREES=${DEVICE_TREES}
-             --build-arg KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME}
-             src/gallium/drivers/panfrost/ci
-    - time docker push $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
-    - docker history $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}
+    GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image
+    IMAGE_LOCAL: "$CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}:latest"
+    DEBIAN_EXEC: 'DEBIAN_ARCH=${DEBIAN_ARCH}
+                  GCC_ARCH=${GCC_ARCH}
+                  KERNEL_ARCH=${KERNEL_ARCH}
+                  DEFCONFIG=${DEFCONFIG}
+                  DEVICE_TREES=${DEVICE_TREES}
+                  KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME}
+                  bash src/gallium/drivers/panfrost/ci/debian-install.sh'
 
 container:armhf:
   extends: .container