From: Tomeu Vizoso Date: Thu, 30 Jul 2020 09:09:38 +0000 (+0200) Subject: ci: Generate MinIO credentials within LAVA jobs X-Git-Url: https://git.libre-soc.org/?p=mesa.git;a=commitdiff_plain;h=e933ac21cb371d38ac2d1cefa90cedf0b00357b2 ci: Generate MinIO credentials within LAVA jobs As these credentials are valid only for 15 minutes, generate them closer to when they are going to be used. Signed-off-by: Tomeu Vizoso Reviewed-by: Daniel Stone Part-of: --- diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml index 0055ef524ee..7501d09233c 100644 --- a/.gitlab-ci/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava-gitlab-ci.yml @@ -43,7 +43,7 @@ kernel+rootfs_armhf: variables: GIT_STRATEGY: none # testing doesn't build anything from source ENV_VARS: "DEQP_PARALLEL=6" - FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1" + FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_JOB_JWT=${CI_JOB_JWT} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1" DEQP_VERSION: gles2 ARTIFACTS_PREFIX: "https://minio-packet.freedesktop.org/mesa-lava/" MESA_URL: "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz" diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index a378b99929e..85b0e5bd098 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -38,10 +38,6 @@ cp -Rp .gitlab-ci/deqp-runner.sh install/ cp -Rp .gitlab-ci/deqp-*-fails.txt install/ cp -Rp .gitlab-ci/deqp-*-skips.txt install/ -ci-fairy minio login $CI_JOB_JWT -# These credentials will be used for uploading artifacts from test jobs -cp .minio_credentials install/ - # Tar up the install dir so that symlinks and hardlinks aren't each # packed separately in the zip file. mkdir -p artifacts/ @@ -55,5 +51,6 @@ if [ -n "$UPLOAD_FOR_LAVA" ]; then gzip -c artifacts/install.tar > mesa-${DEBIAN_ARCH}.tar.gz MINIO_PATH=minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID} + ci-fairy minio login $CI_JOB_JWT ci-fairy minio cp mesa-${DEBIAN_ARCH}.tar.gz minio://${MINIO_PATH}/mesa-${DEBIAN_ARCH}.tar.gz fi diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh index 5c864b9ec1b..246e79cef04 100755 --- a/.gitlab-ci/tracie-runner-gl.sh +++ b/.gitlab-ci/tracie-runner-gl.sh @@ -31,9 +31,6 @@ export PAGER=cat RESULTS=`pwd`/results mkdir -p $RESULTS -# For artifact uploads to MinIO -cp install/.minio_credentials . - # Perform a self-test to ensure tracie is working properly. if [ -z "$TRACIE_NO_UNIT_TESTS" ]; then TRACIE_UPLOAD_TO_MINIO=0 python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py diff --git a/.gitlab-ci/tracie-runner-vk.sh b/.gitlab-ci/tracie-runner-vk.sh index da34bd4d8d4..f046b7c9b40 100755 --- a/.gitlab-ci/tracie-runner-vk.sh +++ b/.gitlab-ci/tracie-runner-vk.sh @@ -20,9 +20,6 @@ export WINEESYNC=1 export DXVK_LOG_LEVEL="none" export DXVK_STATE_CACHE=0 -# For artifact uploads to MinIO -cp install/.minio_credentials . - # Perform a self-test to ensure tracie is working properly. python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py index 0a2c12332d7..2c96bf4ec7d 100644 --- a/.gitlab-ci/tracie/tracie.py +++ b/.gitlab-ci/tracie/tracie.py @@ -13,6 +13,7 @@ import tempfile import time import yaml import shutil +import xml.etree.ElementTree as ET from email.utils import formatdate from pathlib import Path @@ -26,6 +27,8 @@ RESULTS_PATH = "./results/" MINIO_HOST = "minio-packet.freedesktop.org" DASHBOARD_URL = "https://tracie.freedesktop.org/dashboard" +minio_credentials = None + def replay(trace_path, device_name): success = dump_trace_images.dump_from_trace(trace_path, [], device_name) @@ -70,12 +73,38 @@ def sign_with_hmac(key, message): return base64.encodebytes(signature).strip().decode() +def ensure_minio_credentials(): + global minio_credentials + + if minio_credentials is None: + minio_credentials = {} + + params = {'Action': 'AssumeRoleWithWebIdentity', + 'Version': '2011-06-15', + 'RoleArn': 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole', + 'RoleSessionName': '%s:%s' % (os.environ['CI_PROJECT_PATH'], os.environ['CI_JOB_ID']), + 'DurationSeconds': 900, + 'WebIdentityToken': os.environ['CI_JOB_JWT']} + r = requests.post('https://%s' % (MINIO_HOST), params=params) + if r.status_code >= 400: + print(r.text) + r.raise_for_status() + + root = ET.fromstring(r.text) + for attr in root.iter(): + if attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}AccessKeyId': + minio_credentials['AccessKeyId'] = attr.text + elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SecretAccessKey': + minio_credentials['SecretAccessKey'] = attr.text + elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SessionToken': + minio_credentials['SessionToken'] = attr.text + def upload_to_minio(file_name, resource, content_type): - with open('.minio_credentials', 'r') as f: - credentials = json.load(f)[MINIO_HOST] - minio_key = credentials["AccessKeyId"] - minio_secret = credentials["SecretAccessKey"] - minio_token = credentials["SessionToken"] + ensure_minio_credentials() + + minio_key = minio_credentials['AccessKeyId'] + minio_secret = minio_credentials['SecretAccessKey'] + minio_token = minio_credentials['SessionToken'] date = formatdate(timeval=None, localtime=False, usegmt=True) url = 'https://%s%s' % (MINIO_HOST, resource)