From a1947f059fcf118c879752f72e11f6217eb5f5a0 Mon Sep 17 00:00:00 2001 From: Tomeu Vizoso Date: Thu, 9 Jul 2020 22:29:39 +0200 Subject: [PATCH] gitlab-ci: Upload tracie artifacts to MinIO Upload failed images and the results.yml file to MinIO, to facilitate debugging. Also, fix version checking when git is installed as Mesa is going to output a different renderer string if git is installed. Signed-off-by: Tomeu Vizoso Reviewed-by: Daniel Stone Reviewed-By: Rohan Garg Part-of: --- .gitlab-ci.yml | 18 +++++------ .gitlab-ci/container/x86_build-base.sh | 5 ++++ .gitlab-ci/container/x86_build.sh | 1 - .gitlab-ci/lava-gitlab-ci.yml | 3 +- .gitlab-ci/piglit/run.sh | 8 ++++- .gitlab-ci/prepare-artifacts.sh | 8 +++-- .gitlab-ci/tracie-runner-gl.sh | 5 +++- .gitlab-ci/tracie-runner-vk.sh | 3 ++ .gitlab-ci/tracie/README.md | 22 +++++++------- .gitlab-ci/tracie/tracie.py | 41 ++++++++++++++++++++++++++ 10 files changed, 87 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d407b388b24..6ffe2bee5ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,7 +203,7 @@ x86_build-base: - .fdo.container-build@debian - .container variables: - FDO_DISTRIBUTION_TAG: &x86_build-base "2020-06-12" + FDO_DISTRIBUTION_TAG: &x86_build-base "2020-07-21-tracie" .use-x86_build-base: extends: @@ -221,7 +221,7 @@ x86_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &x86_build "2020-07-20-jflags" + FDO_DISTRIBUTION_TAG: &x86_build "2020-07-21-tracie" .use-x86_build: variables: @@ -235,7 +235,7 @@ i386_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &i386_build "2020-07-20-jflags" + FDO_DISTRIBUTION_TAG: &i386_build "2020-07-21-tracie" .use-i386_build: variables: @@ -249,7 +249,7 @@ ppc64el_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-20-jflags" + FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-21-tracie" .use-ppc64el_build: variables: @@ -263,7 +263,7 @@ s390x_build: extends: - .use-x86_build-base variables: - FDO_DISTRIBUTION_TAG: &s390x_build "2020-06-29-jflags" + FDO_DISTRIBUTION_TAG: &s390x_build "2020-07-21-tracie" .use-s390x_build: variables: @@ -276,7 +276,7 @@ s390x_build: x86_test-base: extends: x86_build-base variables: - FDO_DISTRIBUTION_TAG: &x86_test-base "2020-06-02" + FDO_DISTRIBUTION_TAG: &x86_test-base "2020-07-21-tracie" .use-x86_test-base: extends: @@ -293,19 +293,19 @@ x86_test-base: x86_test-gl: extends: .use-x86_test-base variables: - FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-07-15-virgl" + FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-07-21-tracie" # Debian 10 based x86 test image for VK x86_test-vk: extends: .use-x86_test-base variables: - FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-06-18-vkcts" + FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-07-21-tracie" # Debian 9 based x86 build image (old LLVM) x86_build_old: extends: x86_build-base variables: - FDO_DISTRIBUTION_TAG: &x86_build_old "2020-06-29-jflags" + FDO_DISTRIBUTION_TAG: &x86_build_old "2020-07-21-tracie" FDO_DISTRIBUTION_VERSION: stretch-slim .use-x86_build_old: diff --git a/.gitlab-ci/container/x86_build-base.sh b/.gitlab-ci/container/x86_build-base.sh index c904f34ebab..748fea8652b 100644 --- a/.gitlab-ci/container/x86_build-base.sh +++ b/.gitlab-ci/container/x86_build-base.sh @@ -8,6 +8,8 @@ export DEBIAN_FRONTEND=noninteractive apt-get install -y \ ca-certificates \ gnupg \ + python3-pip \ + python3-setuptools \ unzip \ wget @@ -30,6 +32,7 @@ apt-get install -y --no-remove \ g++ \ g++-mingw-w64-x86-64 \ gcc \ + git \ libclang-9-dev \ libclc-dev \ libelf-dev \ @@ -73,6 +76,8 @@ apt-get install -y --no-remove -t buster-backports \ libllvm8 \ meson +# Needed for ci-fairy, this revision is able to upload files to MinIO +pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@6f5af7e5574509726c79109e3c147cee95e81366 # for the vulkan overlay layer wget https://github.com/KhronosGroup/glslang/releases/download/master-tot/glslang-master-linux-Release.zip diff --git a/.gitlab-ci/container/x86_build.sh b/.gitlab-ci/container/x86_build.sh index 9f616538e36..b2220a61faa 100644 --- a/.gitlab-ci/container/x86_build.sh +++ b/.gitlab-ci/container/x86_build.sh @@ -12,7 +12,6 @@ STABLE_EPHEMERAL=" \ autotools-dev \ bzip2 \ cmake \ - git \ gnupg \ libgbm-dev \ libtool \ diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml index dbbf1ea7c09..9bd6f1fe392 100644 --- a/.gitlab-ci/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava-gitlab-ci.yml @@ -33,6 +33,7 @@ kernel+rootfs_armhf: variables: GIT_STRATEGY: none # testing doesn't build anything from source ENV_VARS: "DEQP_PARALLEL=6" + FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1" DEQP_VERSION: gles2 ARTIFACTS_PREFIX: "https://minio-packet.freedesktop.org/mesa-lava/" MESA_URL: "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz" @@ -51,7 +52,7 @@ kernel+rootfs_armhf: --base-artifacts-url ${ARTIFACTS_URL} \ --mesa-url ${MESA_URL} \ --device-type ${DEVICE_TYPE} \ - --env-vars "${ENV_VARS}" \ + --env-vars "${ENV_VARS} ${FIXED_ENV_VARS}" \ --deqp-version ${DEQP_VERSION} \ --kernel-image-name ${KERNEL_IMAGE_NAME} \ --kernel-image-type "${KERNEL_IMAGE_TYPE}" \ diff --git a/.gitlab-ci/piglit/run.sh b/.gitlab-ci/piglit/run.sh index c1f50b9751f..32d2fcd51db 100755 --- a/.gitlab-ci/piglit/run.sh +++ b/.gitlab-ci/piglit/run.sh @@ -8,10 +8,16 @@ VERSION=`cat install/VERSION` cd /piglit PIGLIT_OPTIONS=$(echo $PIGLIT_OPTIONS | head -n 1) +set +e xvfb-run --server-args="-noreset" sh -c \ "export LD_LIBRARY_PATH=$OLDPWD/install/lib; - wflinfo --platform glx --api gl --profile core | grep \"Mesa $VERSION\\\$\" && + wflinfo --platform glx --api gl --profile core | tee /tmp/version.txt | grep \"Mesa $VERSION\\\$\" && ./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results" +retVal=$? +if [ $retVal -ne 0 ]; then + echo "Found $(cat /tmp/version.txt), expected $VERSION" +fi +set -e PIGLIT_RESULTS=${PIGLIT_RESULTS:-$PIGLIT_PROFILES} mkdir -p .gitlab-ci/piglit diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index fb7cc6306d9..40e7bbb5046 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -22,7 +22,7 @@ find install -name \*.so -exec $STRIP {} \; # Test runs don't pull down the git tree, so put the dEQP helper # script and associated bits there. -cp VERSION install/ +echo "$(cat VERSION) (git-$(git rev-parse HEAD | cut -b -10))" >> install/VERSION cp -Rp .gitlab-ci/bare-metal install/ cp -Rp .gitlab-ci/deqp* install/ cp -Rp .gitlab-ci/piglit install/ @@ -41,12 +41,15 @@ cp -Rp .gitlab-ci/deqp-runner.sh install/ cp -Rp .gitlab-ci/deqp-*-fails.txt install/ cp -Rp .gitlab-ci/deqp-*-skips.txt install/ +ci-fairy minio login $CI_JOB_JWT +# These credentials will be used for uploading artifacts from test jobs +cp .minio_credentials install/ + # Tar up the install dir so that symlinks and hardlinks aren't each # packed separately in the zip file. mkdir -p artifacts/ tar -cf artifacts/install.tar install -# If the container has LAVA stuff, prepare the artifacts for LAVA jobs if [ -n "$UPLOAD_FOR_LAVA" ]; then # Pass needed files to the test stage cp $CI_PROJECT_DIR/.gitlab-ci/generate_lava.py artifacts/. @@ -54,7 +57,6 @@ if [ -n "$UPLOAD_FOR_LAVA" ]; then cp $CI_PROJECT_DIR/.gitlab-ci/lava-tracie.yml.jinja2 artifacts/. gzip -c artifacts/install.tar > mesa-${DEBIAN_ARCH}.tar.gz - ci-fairy minio login $CI_JOB_JWT MINIO_PATH=minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID} ci-fairy minio cp mesa-${DEBIAN_ARCH}.tar.gz minio://${MINIO_PATH}/mesa-${DEBIAN_ARCH}.tar.gz fi diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh index 6eb5baadb26..5c864b9ec1b 100755 --- a/.gitlab-ci/tracie-runner-gl.sh +++ b/.gitlab-ci/tracie-runner-gl.sh @@ -31,9 +31,12 @@ export PAGER=cat RESULTS=`pwd`/results mkdir -p $RESULTS +# For artifact uploads to MinIO +cp install/.minio_credentials . + # Perform a self-test to ensure tracie is working properly. if [ -z "$TRACIE_NO_UNIT_TESTS" ]; then - python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py + TRACIE_UPLOAD_TO_MINIO=0 python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py fi if [ "$GALLIUM_DRIVER" = "virpipe" ]; then diff --git a/.gitlab-ci/tracie-runner-vk.sh b/.gitlab-ci/tracie-runner-vk.sh index f046b7c9b40..da34bd4d8d4 100755 --- a/.gitlab-ci/tracie-runner-vk.sh +++ b/.gitlab-ci/tracie-runner-vk.sh @@ -20,6 +20,9 @@ export WINEESYNC=1 export DXVK_LOG_LEVEL="none" export DXVK_STATE_CACHE=0 +# For artifact uploads to MinIO +cp install/.minio_credentials . + # Perform a self-test to ensure tracie is working properly. python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py diff --git a/.gitlab-ci/tracie/README.md b/.gitlab-ci/tracie/README.md index 014bafc92e5..b0526564b46 100644 --- a/.gitlab-ci/tracie/README.md +++ b/.gitlab-ci/tracie/README.md @@ -5,15 +5,13 @@ Home of the Mesa trace testing effort. ### Traces definition file -The trace definition file contains information about the GitLab -project and git commit to get the traces from, and a list of the -traces to run along with their expected image checksums on each -device. An example: +The trace definition file contains information about the traces to run along +with their expected image checksums on each device, and optionally from where to +download them. An example: ```yaml traces-db: - gitlab-project-url: https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db - commit: master + download-url: https://minio-packet.freedesktop.org/mesa-tracie-public/ traces: - path: glmark2/jellyfish.rdc @@ -40,12 +38,10 @@ with the [image_checksum.py](.gitlab-ci/tracie/image_checksum.py) script. Alternatively, an arbitrary checksum can be used, and during replay (see below) the scripts will report the mismatch and expected checksum. -### Trace-db GitLab projects +### Trace-db download urls -The trace-db GitLab projects are assumed to have git repositories -using LFS for their trace files. This is so that trace files can be -potentially checked out and replayed individually, thus reducing -storage requirements during CI runs. +The trace-db:download-url property contains an HTTPS url from which traces can +be downloaded, by appending traces:path properties to it. ### Enabling trace testing on a new device @@ -94,6 +90,10 @@ applications. Traces for proprietary games and application are typically not redistributable, unless specific redistribution rights have been granted by the publisher. +Trace files in a given repository are expected to be immutable once committed +for the first time, so any changes need to be accompanied by a change in the +file name (eg. by appending a _v2 suffix to the file). + ### Replaying traces Mesa traces CI uses a set of scripts to replay traces and check the output diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py index b09bf4a9014..02797c0c887 100644 --- a/.gitlab-ci/tracie/tracie.py +++ b/.gitlab-ci/tracie/tracie.py @@ -1,7 +1,11 @@ import argparse +import base64 +import datetime import enum import glob import hashlib +import hmac +import json import os import requests import sys @@ -10,6 +14,7 @@ import time import yaml import shutil +from email.utils import formatdate from pathlib import Path from PIL import Image from urllib import parse @@ -55,6 +60,38 @@ def gitlab_ensure_trace(project_url, trace): open(trace_path, "wb").write(r.content) print("took %ds." % (time.time() - download_time), flush=True) +def sign_with_hmac(key, message): + key = key.encode("UTF-8") + message = message.encode("UTF-8") + + signature = hmac.new(key, message, hashlib.sha1).digest() + + return base64.encodebytes(signature).strip().decode() + +def upload_artifact(file_name, content_type): + with open('.minio_credentials', 'r') as f: + credentials = json.load(f)["minio-packet.freedesktop.org"] + minio_key = credentials["AccessKeyId"] + minio_secret = credentials["SecretAccessKey"] + minio_token = credentials["SessionToken"] + + resource = '/artifacts/%s/%s/%s' % (os.environ['CI_PROJECT_PATH'], os.environ['CI_PIPELINE_ID'], os.path.basename(file_name)) + date = formatdate(timeval=None, localtime=False, usegmt=True) + url = 'https://minio-packet.freedesktop.org%s' % (resource) + to_sign = "PUT\n\n%s\n%s\nx-amz-security-token:%s\n%s" % (content_type, date, minio_token, resource) + signature = sign_with_hmac(minio_secret, to_sign) + + with open(file_name, 'rb') as data: + headers = {'Host': 'minio-packet.freedesktop.org', + 'Date': date, + 'Content-Type': content_type, + 'Authorization': 'AWS %s:%s' % (minio_key, signature), + 'x-amz-security-token': minio_token} + print("Uploading artifact to %s" % url); + r = requests.put(url, headers=headers, data=data) + #print(r.text) + r.raise_for_status() + def gitlab_check_trace(project_url, device_name, trace, expectation): gitlab_ensure_trace(project_url, trace) @@ -82,6 +119,8 @@ def gitlab_check_trace(project_url, device_name, trace, expectation): results_path = os.path.join(RESULTS_PATH, dir_in_results) os.makedirs(results_path, exist_ok=True) shutil.move(log_file, os.path.join(results_path, os.path.split(log_file)[1])) + if not ok and os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1': + upload_artifact(image_file, 'image/png') if not ok or os.environ.get('TRACIE_STORE_IMAGES', '0') == '1': image_name = os.path.split(image_file)[1] shutil.move(image_file, os.path.join(results_path, image_name)) @@ -116,6 +155,8 @@ def run(filename, device_name): os.makedirs(RESULTS_PATH, exist_ok=True) with open(os.path.join(RESULTS_PATH, 'results.yml'), 'w') as f: yaml.safe_dump(results, f, default_flow_style=False) + if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1': + upload_artifact(os.path.join(RESULTS_PATH, 'results.yml'), 'text/yaml') return all_ok -- 2.30.2