From: Tomeu Vizoso Date: Fri, 7 Jun 2019 08:20:28 +0000 (+0200) Subject: panfrost: ci: Switch from direct Docker use to buildah X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=2fe7f9f2aefc62bcd16e9bcbeaa0c9862d0b006a;p=mesa.git panfrost: ci: Switch from direct Docker use to buildah Use the infrastructure in wayland/ci-templates to build the container images. This prevents from getting into some situations in which the images wouldn't be rebuilt, and allows us to share some infrastructure with other projects in freedesktop.org. Signed-off-by: Tomeu Vizoso Suggested-by: Michel Dänzer Acked-by: Alyssa Rosenzweig --- diff --git a/src/gallium/drivers/panfrost/ci/Dockerfile b/src/gallium/drivers/panfrost/ci/Dockerfile deleted file mode 100644 index fe0e67c27bd..00000000000 --- a/src/gallium/drivers/panfrost/ci/Dockerfile +++ /dev/null @@ -1,135 +0,0 @@ -FROM debian:testing - -ENV DEBIAN_FRONTEND=noninteractive - -RUN echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft -RUN echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft -RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d -RUN echo 'exit 101' >> /usr/sbin/policy-rc.d -RUN chmod +x /usr/sbin/policy-rc.d - -############### Install packages for building - -ARG DEBIAN_ARCH -RUN dpkg --add-architecture ${DEBIAN_ARCH} -RUN echo deb-src http://deb.debian.org/debian testing main >> /etc/apt/sources.list -RUN apt-get update && \ - apt-get -y install ca-certificates && \ - apt-get -y install --no-install-recommends \ - crossbuild-essential-${DEBIAN_ARCH} \ - meson \ - g++ \ - git \ - ccache \ - pkg-config \ - python3-mako \ - python-numpy \ - python-six \ - python-mako \ - python3-pip \ - python3-setuptools \ - python3-six \ - python3-wheel \ - python3-jinja2 \ - bison \ - flex \ - libwayland-dev \ - gettext \ - cmake \ - bc \ - libssl-dev \ - lavacli \ - csvkit \ - curl \ - unzip \ - wget \ - debootstrap \ - procps \ - qemu-user-static \ - cpio \ - \ - libdrm-dev:${DEBIAN_ARCH} \ - libx11-dev:${DEBIAN_ARCH} \ - libxxf86vm-dev:${DEBIAN_ARCH} \ - libexpat1-dev:${DEBIAN_ARCH} \ - libsensors-dev:${DEBIAN_ARCH} \ - libxfixes-dev:${DEBIAN_ARCH} \ - libxdamage-dev:${DEBIAN_ARCH} \ - libxext-dev:${DEBIAN_ARCH} \ - x11proto-dev:${DEBIAN_ARCH} \ - libx11-xcb-dev:${DEBIAN_ARCH} \ - libxcb-dri2-0-dev:${DEBIAN_ARCH} \ - libxcb-glx0-dev:${DEBIAN_ARCH} \ - libxcb-xfixes0-dev:${DEBIAN_ARCH} \ - libxcb-dri3-dev:${DEBIAN_ARCH} \ - libxcb-present-dev:${DEBIAN_ARCH} \ - libxcb-randr0-dev:${DEBIAN_ARCH} \ - libxcb-sync-dev:${DEBIAN_ARCH} \ - libxrandr-dev:${DEBIAN_ARCH} \ - libxshmfence-dev:${DEBIAN_ARCH} \ - libelf-dev:${DEBIAN_ARCH} \ - libwayland-dev:${DEBIAN_ARCH} \ - libwayland-egl-backend-dev:${DEBIAN_ARCH} \ - libclang-7-dev:${DEBIAN_ARCH} \ - zlib1g-dev:${DEBIAN_ARCH} \ - libglvnd-core-dev:${DEBIAN_ARCH} \ - wayland-protocols:${DEBIAN_ARCH} \ - libpng-dev:${DEBIAN_ARCH} && \ - rm -rf /var/lib/apt/lists - -############### Cross-build dEQP -ARG GCC_ARCH -RUN mkdir -p /artifacts/rootfs/deqp && \ - wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip && \ - unzip -q opengl-es-cts-3.2.5.0.zip -d / && \ - rm opengl-es-cts-3.2.5.0.zip && \ - cd /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \ - python3 external/fetch_sources.py && \ - cd /artifacts/rootfs/deqp && \ - cmake -DDEQP_TARGET=wayland \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \ - -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \ - /VK-GL-CTS-opengl-es-cts-3.2.5.0 && \ - make -j$(nproc) && \ - rm -rf /artifacts/rootfs/deqp/external && \ - rm -rf /artifacts/rootfs/deqp/modules/gles3 && \ - rm -rf /artifacts/rootfs/deqp/modules/gles31 && \ - rm -rf /artifacts/rootfs/deqp/modules/internal && \ - rm -rf /artifacts/rootfs/deqp/executor && \ - rm -rf /artifacts/rootfs/deqp/execserver && \ - rm -rf /artifacts/rootfs/deqp/modules/egl && \ - rm -rf /artifacts/rootfs/deqp/framework && \ - find . -name CMakeFiles | xargs rm -rf && \ - find . -name lib\*.a | xargs rm -rf && \ - du -sh * && \ - rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0 - - -############### Cross-build kernel - -ARG KERNEL_ARCH -ARG DEFCONFIG -ARG DEVICE_TREES -ARG KERNEL_IMAGE_NAME -ENV KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz" - -COPY ${KERNEL_ARCH}.config /panfrost-ci/ -RUN mkdir -p /kernel && \ - wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel && \ - cd /kernel && \ - ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" ./scripts/kconfig/merge_config.sh ${DEFCONFIG} /panfrost-ci/${KERNEL_ARCH}.config && \ - ARCH=${KERNEL_ARCH} CROSS_COMPILE="${GCC_ARCH}-" make -j12 ${KERNEL_IMAGE_NAME} dtbs && \ - cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/. && \ - cp ${DEVICE_TREES} /artifacts/. && \ - rm -rf /kernel - - -############### Create rootfs - -COPY create-rootfs.sh /artifacts/rootfs/ -RUN debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian && \ - chroot /artifacts/rootfs sh /create-rootfs.sh && \ - rm /artifacts/rootfs/create-rootfs.sh - -ENTRYPOINT [""] diff --git a/src/gallium/drivers/panfrost/ci/debian-install.sh b/src/gallium/drivers/panfrost/ci/debian-install.sh new file mode 100644 index 00000000000..8a8b4ec9a47 --- /dev/null +++ b/src/gallium/drivers/panfrost/ci/debian-install.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +set -e +set -o xtrace + +PANFROST_CI_DIR=/tmp/clone/src/gallium/drivers/panfrost/ci + +############### Install packages for building +dpkg --add-architecture ${DEBIAN_ARCH} +echo 'deb-src https://deb.debian.org/debian testing main' > /etc/apt/sources.list.d/deb-src.list +apt-get update +apt-get -y install ca-certificates +apt-get -y install --no-install-recommends \ + crossbuild-essential-${DEBIAN_ARCH} \ + meson \ + g++ \ + git \ + ccache \ + pkg-config \ + python3-mako \ + python-numpy \ + python-six \ + python-mako \ + python3-pip \ + python3-setuptools \ + python3-six \ + python3-wheel \ + python3-jinja2 \ + bison \ + flex \ + libwayland-dev \ + gettext \ + cmake \ + bc \ + libssl-dev \ + lavacli \ + csvkit \ + curl \ + unzip \ + wget \ + debootstrap \ + procps \ + qemu-user-static \ + cpio \ + \ + libdrm-dev:${DEBIAN_ARCH} \ + libx11-dev:${DEBIAN_ARCH} \ + libxxf86vm-dev:${DEBIAN_ARCH} \ + libexpat1-dev:${DEBIAN_ARCH} \ + libsensors-dev:${DEBIAN_ARCH} \ + libxfixes-dev:${DEBIAN_ARCH} \ + libxdamage-dev:${DEBIAN_ARCH} \ + libxext-dev:${DEBIAN_ARCH} \ + x11proto-dev:${DEBIAN_ARCH} \ + libx11-xcb-dev:${DEBIAN_ARCH} \ + libxcb-dri2-0-dev:${DEBIAN_ARCH} \ + libxcb-glx0-dev:${DEBIAN_ARCH} \ + libxcb-xfixes0-dev:${DEBIAN_ARCH} \ + libxcb-dri3-dev:${DEBIAN_ARCH} \ + libxcb-present-dev:${DEBIAN_ARCH} \ + libxcb-randr0-dev:${DEBIAN_ARCH} \ + libxcb-sync-dev:${DEBIAN_ARCH} \ + libxrandr-dev:${DEBIAN_ARCH} \ + libxshmfence-dev:${DEBIAN_ARCH} \ + libelf-dev:${DEBIAN_ARCH} \ + libwayland-dev:${DEBIAN_ARCH} \ + libwayland-egl-backend-dev:${DEBIAN_ARCH} \ + libclang-7-dev:${DEBIAN_ARCH} \ + zlib1g-dev:${DEBIAN_ARCH} \ + libglvnd-core-dev:${DEBIAN_ARCH} \ + wayland-protocols:${DEBIAN_ARCH} \ + libpng-dev:${DEBIAN_ARCH} + +############### Cross-build dEQP +mkdir -p /artifacts/rootfs/deqp + +wget https://github.com/KhronosGroup/VK-GL-CTS/archive/opengl-es-cts-3.2.5.0.zip +unzip -q opengl-es-cts-3.2.5.0.zip -d / +rm opengl-es-cts-3.2.5.0.zip + +cd /VK-GL-CTS-opengl-es-cts-3.2.5.0 +python3 external/fetch_sources.py + +cd /artifacts/rootfs/deqp +cmake -DDEQP_TARGET=wayland \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=${GCC_ARCH}-gcc \ + -DCMAKE_CXX_COMPILER=${GCC_ARCH}-g++ \ + /VK-GL-CTS-opengl-es-cts-3.2.5.0 +make -j$(nproc) +rm -rf /artifacts/rootfs/deqp/external +rm -rf /artifacts/rootfs/deqp/modules/gles3 +rm -rf /artifacts/rootfs/deqp/modules/gles31 +rm -rf /artifacts/rootfs/deqp/modules/internal +rm -rf /artifacts/rootfs/deqp/executor +rm -rf /artifacts/rootfs/deqp/execserver +rm -rf /artifacts/rootfs/deqp/modules/egl +rm -rf /artifacts/rootfs/deqp/framework +find . -name CMakeFiles | xargs rm -rf +find . -name lib\*.a | xargs rm -rf +du -sh * +rm -rf /VK-GL-CTS-opengl-es-cts-3.2.5.0 + + +############### Cross-build kernel +KERNEL_URL="https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+archive/refs/tags/v5.2-rc2.tar.gz" +export ARCH=${KERNEL_ARCH} +export CROSS_COMPILE="${GCC_ARCH}-" + +mkdir -p /kernel +wget -qO- ${KERNEL_URL} | tar -xvz -C /kernel +cd /kernel +./scripts/kconfig/merge_config.sh ${DEFCONFIG} ${PANFROST_CI_DIR}/${KERNEL_ARCH}.config +make -j12 ${KERNEL_IMAGE_NAME} dtbs +cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /artifacts/. +cp ${DEVICE_TREES} /artifacts/. +rm -rf /kernel + + +############### Create rootfs +cp ${PANFROST_CI_DIR}/create-rootfs.sh /artifacts/rootfs/. +debootstrap --variant=minbase --arch=${DEBIAN_ARCH} testing /artifacts/rootfs/ http://deb.debian.org/debian +chroot /artifacts/rootfs sh /create-rootfs.sh +rm /artifacts/rootfs/create-rootfs.sh + diff --git a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml index 3491b9f9875..d67c1ff6c94 100644 --- a/src/gallium/drivers/panfrost/ci/gitlab-ci.yml +++ b/src/gallium/drivers/panfrost/ci/gitlab-ci.yml @@ -1,39 +1,55 @@ +# DEBIAN_TAG is the tag of the docker image used for the build jobs. If the +# image doesn't exist yet, the docker-image stage generates it. +# +# In order to generate a new image, one should generally change the tag. +# While removing the image from the registry would also work, that's not +# recommended except for ephemeral images during development: Replacing an +# image after a significant amount of time might pull in newer versions of +# gcc/clang or other packages, which might break the build with older commits +# using the same tag. +# +# After merging a change resulting in generating a new image to the main +# repository, it's recommended to remove the image from the source repository's +# container registry, so that the image from the main repository's registry +# will be used there as well. You can manage your images on your fork of: +# https://gitlab.freedesktop.org/xorg/xserver/container_registry variables: - DOCKER_DRIVER: overlay2 + UPSTREAM_REPO: mesa/mesa + DEBIAN_VERSION: testing-slim + DEBIAN_TAG: "2019-06-07-1" + +include: + - project: 'wayland/ci-templates' + ref: c73dae8b84697ef18e2dbbf4fed7386d9652b0cd + file: '/templates/debian.yml' stages: - containers - build - test +# Retry jobs after runner system failures +.retry: &retry + retry: + max: 2 + when: + - runner_system_failure + # Build Docker image with deqp, the rootfs and the build deps for Mesa .container: + extends: .debian@container-ifnot-exists stage: containers - image: docker:stable - only: - changes: - - src/gallium/drivers/panfrost/ci/gitlab-ci.yml - - src/gallium/drivers/panfrost/ci/Dockerfile - - src/gallium/drivers/panfrost/ci/create-rootfs.sh - - src/gallium/drivers/panfrost/ci/${KERNEL_ARCH}.config - services: - - docker:dind + <<: *retry variables: - DOCKER_HOST: tcp://docker:2375 - DOCKER_DRIVER: overlay2 - script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - time docker build - -t $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH} - --build-arg DEBIAN_ARCH=${DEBIAN_ARCH} - --build-arg GCC_ARCH=${GCC_ARCH} - --build-arg KERNEL_ARCH=${KERNEL_ARCH} - --build-arg DEFCONFIG=${DEFCONFIG} - --build-arg DEVICE_TREES=${DEVICE_TREES} - --build-arg KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME} - src/gallium/drivers/panfrost/ci - - time docker push $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH} - - docker history $CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH} + GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image + IMAGE_LOCAL: "$CI_REGISTRY/$CI_PROJECT_PATH/${DEBIAN_ARCH}:latest" + DEBIAN_EXEC: 'DEBIAN_ARCH=${DEBIAN_ARCH} + GCC_ARCH=${GCC_ARCH} + KERNEL_ARCH=${KERNEL_ARCH} + DEFCONFIG=${DEFCONFIG} + DEVICE_TREES=${DEVICE_TREES} + KERNEL_IMAGE_NAME=${KERNEL_IMAGE_NAME} + bash src/gallium/drivers/panfrost/ci/debian-install.sh' container:armhf: extends: .container