ci: Generate MinIO credentials within LAVA jobs

As these credentials are valid only for 15 minutes, generate them closer
to when they are going to be used.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Daniel Stone <daniels@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6124>

diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index 0055ef524ee138e19a40a40f7a44b41741413ab6..7501d09233cda728827fda6e3fae6b273e04601c 100644 (file)
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -43,7 +43,7 @@ kernel+rootfs_armhf:
   variables:
     GIT_STRATEGY: none # testing doesn't build anything from source
     ENV_VARS: "DEQP_PARALLEL=6"
-    FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1"
+    FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_JOB_JWT=${CI_JOB_JWT} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1"
     DEQP_VERSION: gles2
     ARTIFACTS_PREFIX: "https://minio-packet.freedesktop.org/mesa-lava/"
     MESA_URL: "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"

diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index a378b99929ec97fe8c0e527672096703e0a89e4c..85b0e5bd098c41d9d3c7a78e1c94181254af3580 100755 (executable)
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -38,10 +38,6 @@ cp -Rp .gitlab-ci/deqp-runner.sh install/
 cp -Rp .gitlab-ci/deqp-*-fails.txt install/
 cp -Rp .gitlab-ci/deqp-*-skips.txt install/
 
-ci-fairy minio login $CI_JOB_JWT
-# These credentials will be used for uploading artifacts from test jobs
-cp .minio_credentials install/
-
 # Tar up the install dir so that symlinks and hardlinks aren't each
 # packed separately in the zip file.
 mkdir -p artifacts/
@@ -55,5 +51,6 @@ if [ -n "$UPLOAD_FOR_LAVA" ]; then
 
     gzip -c artifacts/install.tar > mesa-${DEBIAN_ARCH}.tar.gz
     MINIO_PATH=minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
+    ci-fairy minio login $CI_JOB_JWT
     ci-fairy minio cp mesa-${DEBIAN_ARCH}.tar.gz minio://${MINIO_PATH}/mesa-${DEBIAN_ARCH}.tar.gz
 fi

diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh
index 5c864b9ec1b89fff0b747cca6322f1aa4742559e..246e79cef048273727db436f5718b58b84b135f5 100755 (executable)
--- a/.gitlab-ci/tracie-runner-gl.sh
+++ b/.gitlab-ci/tracie-runner-gl.sh
@@ -31,9 +31,6 @@ export PAGER=cat
 RESULTS=`pwd`/results
 mkdir -p $RESULTS
 
-# For artifact uploads to MinIO
-cp install/.minio_credentials .
-
 # Perform a self-test to ensure tracie is working properly.
 if [ -z "$TRACIE_NO_UNIT_TESTS" ]; then
     TRACIE_UPLOAD_TO_MINIO=0 python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py

diff --git a/.gitlab-ci/tracie-runner-vk.sh b/.gitlab-ci/tracie-runner-vk.sh
index da34bd4d8d44063dba86c515f4b9f21f4315df9a..f046b7c9b4052f571f6954708b5282e21dca3ba1 100755 (executable)
--- a/.gitlab-ci/tracie-runner-vk.sh
+++ b/.gitlab-ci/tracie-runner-vk.sh
@@ -20,9 +20,6 @@ export WINEESYNC=1
 export DXVK_LOG_LEVEL="none"
 export DXVK_STATE_CACHE=0
 
-# For artifact uploads to MinIO
-cp install/.minio_credentials .
-
 # Perform a self-test to ensure tracie is working properly.
 python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
 

diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py
index 0a2c12332d724cf704691889cac27c4980887624..2c96bf4ec7de1df8854a5074d46c1a37121f32c4 100644 (file)
--- a/.gitlab-ci/tracie/tracie.py
+++ b/.gitlab-ci/tracie/tracie.py
@@ -13,6 +13,7 @@ import tempfile
 import time
 import yaml
 import shutil
+import xml.etree.ElementTree as ET
 
 from email.utils import formatdate
 from pathlib import Path
@@ -26,6 +27,8 @@ RESULTS_PATH = "./results/"
 MINIO_HOST = "minio-packet.freedesktop.org"
 DASHBOARD_URL = "https://tracie.freedesktop.org/dashboard"
 
+minio_credentials = None
+
 def replay(trace_path, device_name):
     success = dump_trace_images.dump_from_trace(trace_path, [], device_name)
 
@@ -70,12 +73,38 @@ def sign_with_hmac(key, message):
 
     return base64.encodebytes(signature).strip().decode()
 
+def ensure_minio_credentials():
+    global minio_credentials
+
+    if minio_credentials is None:
+        minio_credentials = {}
+
+    params = {'Action': 'AssumeRoleWithWebIdentity',
+              'Version': '2011-06-15',
+              'RoleArn': 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole',
+              'RoleSessionName': '%s:%s' % (os.environ['CI_PROJECT_PATH'], os.environ['CI_JOB_ID']),
+              'DurationSeconds': 900,
+              'WebIdentityToken': os.environ['CI_JOB_JWT']}
+    r = requests.post('https://%s' % (MINIO_HOST), params=params)
+    if r.status_code >= 400:
+        print(r.text)
+    r.raise_for_status()
+
+    root = ET.fromstring(r.text)
+    for attr in root.iter():
+        if attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}AccessKeyId':
+            minio_credentials['AccessKeyId'] = attr.text
+        elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SecretAccessKey':
+            minio_credentials['SecretAccessKey'] = attr.text
+        elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SessionToken':
+            minio_credentials['SessionToken'] = attr.text
+
 def upload_to_minio(file_name, resource, content_type):
-    with open('.minio_credentials', 'r') as f:
-        credentials = json.load(f)[MINIO_HOST]
-        minio_key = credentials["AccessKeyId"]
-        minio_secret = credentials["SecretAccessKey"]
-        minio_token = credentials["SessionToken"]
+    ensure_minio_credentials()
+
+    minio_key = minio_credentials['AccessKeyId']
+    minio_secret = minio_credentials['SecretAccessKey']
+    minio_token = minio_credentials['SessionToken']
 
     date = formatdate(timeval=None, localtime=False, usegmt=True)
     url = 'https://%s%s' % (MINIO_HOST, resource)