ci: Include trace replay support in ARM rootfses.

Builds the renderdoc and apitrace programs so we can replay GL traces on
DUTs.

[Separated out from 5472's commit that also enabled the jobs in LAVA,
dropped unnecessary python packages from arm_build, fixed up arm64_test
build, traces-db in baremetal, new commit message by anholt]

Signed-off-by: Rohan Garg <rohan.garg@collabora.com>
Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Eric Anholt <eric@anholt.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5433>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59ebcf4015a0288ae98c6e362a55be03f771aeec..a59cfcd44580d407b156db02378eb29bf3b46bb5 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -272,7 +272,7 @@ arm_build:
     - .fdo.container-build@debian@arm64v8
     - .container
   variables:
-    FDO_DISTRIBUTION_TAG: &arm_build "2020-06-22-stable"
+    FDO_DISTRIBUTION_TAG: &arm_build "2020-06-22-tracie"
 
 .use-arm_build:
   variables:
@@ -287,7 +287,7 @@ arm_test-base:
     - .fdo.container-build@debian
     - .container
   variables:
-    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-06-22-stable"
+    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-06-22-tracie"
 
 .use-arm_test-base:
   extends:
@@ -305,7 +305,7 @@ arm64_test:
   extends:
     - .use-arm_test-base
   variables:
-    FDO_DISTRIBUTION_TAG: &arm64_test "2020-06-22-ephemeral"
+    FDO_DISTRIBUTION_TAG: &arm64_test "2020-06-22-tracie"
 
 .use-arm64_test:
   variables:

diff --git a/.gitlab-ci/bare-metal/fastboot.sh b/.gitlab-ci/bare-metal/fastboot.sh
index 9a4e6f649594df96e7ddfc2ba6cf96ac1d8562c5..ad4a665055b56e9b5bc7eba0e0442bd7710cc958 100755 (executable)
--- a/.gitlab-ci/bare-metal/fastboot.sh
+++ b/.gitlab-ci/bare-metal/fastboot.sh
@@ -60,7 +60,8 @@ rsync -a --delete $BM_ROOTFS/ rootfs/
 # these devices use it and it would take up space in the initrd.
 pushd rootfs
 find -H | \
-  egrep -v "external/(openglcts|vulkancts|amber|glslang|spirv-tools)" | \
+  egrep -v "external/(openglcts|vulkancts|amber|glslang|spirv-tools)" |
+  egrep -v "traces-db|apitrace|renderdoc|python" | \
   cpio -H newc -o | \
   xz --check=crc32 -T4 - > $CI_PROJECT_DIR/rootfs.cpio.gz
 popd

diff --git a/.gitlab-ci/build-apitrace.sh b/.gitlab-ci/build-apitrace.sh
index 87582f3b9e655e3873e50a6aceb48439ad425905..8b44b66a2cea5b3385d1c66ff0ca998d77fb1cb2 100644 (file)
--- a/.gitlab-ci/build-apitrace.sh
+++ b/.gitlab-ci/build-apitrace.sh
@@ -9,22 +9,25 @@ WAFFLE_VERSION="e3c995d9a2693b687501715b6550619922346089"
 git clone https://gitlab.freedesktop.org/mesa/waffle.git --single-branch --no-checkout /waffle
 pushd /waffle
 git checkout "$WAFFLE_VERSION"
-cmake -B_build -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=Release .
-make -C _build -j4 install
+cmake -B_build -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=Release $EXTRA_CMAKE_ARGS .
+make -C _build install
+mkdir -p build/lib build/bin
+cp _build/lib/libwaffle-1.so build/lib/libwaffle-1.so.0
+cp _build/bin/wflinfo build/bin/wflinfo
+${STRIP_CMD:-strip} build/lib/* build/bin/*
+find . -not -path './build' -not -path './build/*' -delete
 popd
-rm -rf /waffle
 
 APITRACE_VERSION="9.0"
 
 git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout /apitrace
 pushd /apitrace
 git checkout "$APITRACE_VERSION"
-cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/
+cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/ $EXTRA_CMAKE_ARGS
 ninja -C _build
 mkdir build
 cp _build/apitrace build
-cp _build/glretrace build
 cp _build/eglretrace build
-strip build/*
+${STRIP_CMD:-strip} build/*
 find . -not -path './build' -not -path './build/*' -delete
 popd

diff --git a/.gitlab-ci/build-renderdoc.sh b/.gitlab-ci/build-renderdoc.sh
index 5d023c7362771e61b0b066e52d9803b9958128b5..bc21cc864a9c08020160135ab25e77afe33690cf 100644 (file)
--- a/.gitlab-ci/build-renderdoc.sh
+++ b/.gitlab-ci/build-renderdoc.sh
@@ -7,11 +7,11 @@ RENDERDOC_VERSION=da02e88201dc3b64316fc33ce6ff69cc729689aa
 git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout /renderdoc
 pushd /renderdoc
 git checkout "$RENDERDOC_VERSION"
-cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release
+cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release $EXTRA_CMAKE_ARGS
 ninja -C _build
 mkdir -p build/lib
+${STRIP_CMD:-strip} _build/lib/*.so
 cp _build/lib/renderdoc.so build/lib
 cp _build/lib/librenderdoc.so build/lib
-strip build/lib/*
 find . -not -path './build' -not -path './build/*' -delete
 popd

diff --git a/.gitlab-ci/container/arm_build.sh b/.gitlab-ci/container/arm_build.sh
index 6facab31e734b3de7f514c044fcadf9c60fcadac..4266aee2ba8654808183b9c3b4c40bb5318ae645 100644 (file)
--- a/.gitlab-ci/container/arm_build.sh
+++ b/.gitlab-ci/container/arm_build.sh
@@ -12,6 +12,8 @@ apt-get update
 apt-get -y install \
        abootimg \
        android-sdk-ext4-utils \
+       autoconf \
+       automake \
        bc \
        bison \
        ccache \
@@ -24,28 +26,45 @@ apt-get -y install \
        g++ \
        git \
        lavacli \
+       libboost-dev:armhf \
+       libboost-dev \
        libdrm-dev:armhf \
+       libdrm-dev \
        libegl1-mesa-dev \
        libegl1-mesa-dev:armhf \
        libelf-dev \
        libelf-dev:armhf \
        libexpat1-dev \
        libexpat1-dev:armhf \
+       libgbm-dev \
+       libgbm-dev:armhf \
        libgles2-mesa-dev \
        libgles2-mesa-dev:armhf \
+       libpcre3-dev \
+       libpcre3-dev:armhf \
        libpng-dev \
        libpng-dev:armhf \
+       libpython3-dev \
+       libpython3-dev:armhf \
        libssl-dev \
        libvulkan-dev \
+       libvulkan-dev \
        libvulkan-dev:armhf \
+       libxcb-keysyms1-dev \
+       libxcb-keysyms1-dev:armhf \
        llvm-7-dev:armhf \
        llvm-8-dev \
        pkg-config \
        python \
+       python3-dev \
        python3-distutils \
        python3-setuptools \
        python3-mako \
        python3-serial \
+       qt5-default \
+       qt5-qmake \
+       qtbase5-dev \
+       qtbase5-dev:armhf \
        unzip \
        wget \
        xz-utils \

diff --git a/.gitlab-ci/container/baremetal_build.sh b/.gitlab-ci/container/baremetal_build.sh
index b84608f87e9eb1b1f44fc547c8619025e19a0d78..6024138f8abd4a70a5e58fad62b181213d0049f2 100644 (file)
--- a/.gitlab-ci/container/baremetal_build.sh
+++ b/.gitlab-ci/container/baremetal_build.sh
@@ -3,24 +3,38 @@
 set -e
 set -o xtrace
 
+ROOTFS=/lava-files/rootfs-${arch}
+
 dpkg --add-architecture $arch
 apt-get update
 
 # Cross-build test deps
 BAREMETAL_EPHEMERAL=" \
+        autoconf \
+        automake \
         crossbuild-essential-$arch \
+        git-lfs \
         libdrm-dev:$arch \
+        libboost-dev:$arch \
         libegl1-mesa-dev:$arch \
         libelf-dev:$arch \
         libexpat1-dev:$arch \
         libffi-dev:$arch \
         libgbm-dev:$arch \
         libgles2-mesa-dev:$arch \
+        libpcre3-dev:$arch \
         libpng-dev:$arch \
+        libpython3-dev:$arch \
         libstdc++6:$arch \
         libtinfo-dev:$arch \
         libegl1-mesa-dev:$arch \
-        libvulkan-dev:$arch
+        libvulkan-dev:$arch \
+        libxcb-keysyms1-dev:$arch \
+        libpython3-dev:$arch \
+        python3-dev \
+        qt5-default \
+        qt5-qmake \
+        qtbase5-dev:$arch \
         "
 
 apt-get install -y --no-remove $BAREMETAL_EPHEMERAL
@@ -37,6 +51,20 @@ mkdir /var/cache/apt/archives/$arch
 
 DEBIAN_ARCH=$arch INCLUDE_VK_CTS=1 . .gitlab-ci/container/lava_arm.sh
 
+############### Store traces
+# Clone the traces-db at container build time so we don't have to pull traces
+# per run (too much egress cost for fd.o).
+git clone \
+    --depth 1 \
+    -b mesa-ci-2020-06-08 \
+    https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db.git \
+    $ROOTFS/traces-db
+rm -rf $ROOTFS/traces-db/.git
+find $ROOTFS/traces-db -type f \
+     -a -not -name '*.trace' \
+     -a -not -name '*.rdc' \
+     -delete
+
 ccache --show-stats
 
 . .gitlab-ci/container/container_post_build.sh

diff --git a/.gitlab-ci/container/lava_arm.sh b/.gitlab-ci/container/lava_arm.sh
index 05bc602aa6f9a15c25897fa1b13bfccf654b2ba3..ed1a10eaec5fe9b2331b76ba34e4e007b6996801 100644 (file)
--- a/.gitlab-ci/container/lava_arm.sh
+++ b/.gitlab-ci/container/lava_arm.sh
@@ -43,6 +43,25 @@ fi
 mv /deqp /lava-files/rootfs-${DEBIAN_ARCH}/.
 
 
+############### Build apitrace
+. .gitlab-ci/build-apitrace.sh
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
+mv /apitrace/build /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
+rm -rf /apitrace
+
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/waffle
+mv /waffle/build /lava-files/rootfs-${DEBIAN_ARCH}/waffle
+rm -rf /waffle
+
+
+############### Build renderdoc
+EXTRA_CMAKE_ARGS+=" -DENABLE_XCB=false"
+. .gitlab-ci/build-renderdoc.sh
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
+mv /renderdoc/build /lava-files/rootfs-${DEBIAN_ARCH}/renderdoc
+rm -rf /renderdoc
+
+
 ############### Cross-build kernel
 KERNEL_URL="https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.5-panfrost-fixes/linux-v5.5-panfrost-fixes.tar.gz"
 

diff --git a/.gitlab-ci/create-rootfs.sh b/.gitlab-ci/create-rootfs.sh
index 46a3dd239df6b1897b439470bbdb6551ef5c12bf..ef1026fea7cc8d64587e15848e51cccdd2971a57 100644 (file)
--- a/.gitlab-ci/create-rootfs.sh
+++ b/.gitlab-ci/create-rootfs.sh
@@ -11,8 +11,16 @@ apt-get -y install --no-install-recommends \
     libexpat1 \
     libdrm2 \
     libdrm-nouveau2 \
+    libx11-6 \
+    libx11-xcb1 \
     firmware-qcom-media \
     netcat-openbsd \
+    python3 \
+    libpython3.7 \
+    python3-pil \
+    python3-pytest \
+    python3-requests \
+    python3-yaml \
     wget \
     xz-utils
 
@@ -72,6 +80,7 @@ rm -rf /var/log/*
 # Dropping documentation, localization, i18n files, etc
 rm -rf /usr/share/doc/*
 rm -rf /usr/share/locale/*
+rm -rf /usr/share/X11/locale/*
 rm -rf /usr/share/man
 rm -rf /usr/share/i18n/*
 rm -rf /usr/share/info/*
@@ -115,10 +124,20 @@ UNNEEDED_PACKAGES="apt libapt-pkg6.0 "\
 "passwd "\
 "libsemanage1 libsemanage-common "\
 "libsepol1 "\
+"gzip "\
 "gpgv "\
 "hostname "\
 "adduser "\
 "debian-archive-keyring "\
+"libegl1-mesa-dev "\
+"libegl-mesa0 "\
+"libgl1-mesa-dev "\
+"libgl1-mesa-dri "\
+"libglapi-mesa "\
+"libgles2-mesa-dev "\
+"libglx-mesa0 "\
+"mesa-common-dev "\
+"libz3-4 "\
 
 # Removing unneeded packages
 for PACKAGE in ${UNNEEDED_PACKAGES}

diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh
index 426333ce29553ef5d3e202d8ef53e0ae6467021b..5c47114e43f052d083e901d885ba9f742ec19243 100755 (executable)
--- a/.gitlab-ci/tracie-runner-gl.sh
+++ b/.gitlab-ci/tracie-runner-gl.sh
@@ -17,6 +17,9 @@ export LD_LIBRARY_PATH="/waffle/build/lib:$LD_LIBRARY_PATH"
 # Set environment for apitrace executable.
 export PATH="/apitrace/build:$PATH"
 
+# Set environment for wflinfo executable.
+export PATH="/waffle/build/bin:$PATH"
+
 # Use the surfaceless EGL platform.
 export EGL_PLATFORM="surfaceless"
 export DISPLAY=