gitlab-ci: Download traces from MinIO

Downloading the traces directly from git causes very high egress from
GCE, which is expensive.

So we can expand trace testing further, we are going to keep a cache in
freedesktop.org's MinIO instance. This commit implements downloading
from it.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Daniel Stone <daniels@collabora.com>
Reviewed-By: Rohan Garg <rohan.garg@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5472>

diff --git a/.gitlab-ci/traces.yml b/.gitlab-ci/traces.yml
index 1ec7603407f71d8e5b9771eba8ae3760914ceb66..87db9e948569d87167a4b8a761e9a99c35a4e5c9 100644 (file)
--- a/.gitlab-ci/traces.yml
+++ b/.gitlab-ci/traces.yml
@@ -1,6 +1,5 @@
 traces-db:
-  gitlab-project-url: "https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db"
-  commit: "d3b1efe0cd69ef6ae40a29a14ed733ee0ba0cb4c"
+  download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
 
 traces:
   - path: glmark2/desktop-blur-radius=5:effect=blur:passes=1:separable=true:windows=4.rdc

diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py
index 59016cae164c13f916442bfb143771b3be974f31..b09bf4a90145f8b49dbf2cea3ae27ed230ab9df7 100644 (file)
--- a/.gitlab-ci/tracie/tracie.py
+++ b/.gitlab-ci/tracie/tracie.py
@@ -36,55 +36,9 @@ def replay(trace_path, device_name):
         log_file = files[0]
         return hashlib.md5(Image.open(image_file).tobytes()).hexdigest(), image_file, log_file
 
-def gitlab_download_metadata(project_url, repo_commit, trace_path):
-    url = parse.urlparse(project_url)
-
-    url_path = url.path
-    if url_path.startswith("/"):
-        url_path = url_path[1:]
-
-    gitlab_api_url = url.scheme + "://" + url.netloc + "/api/v4/projects/" + parse.quote_plus(url_path)
-
-    r = requests.get(gitlab_api_url + "/repository/files/%s/raw?ref=%s" % (parse.quote_plus(trace_path), repo_commit))
-    metadata_raw = r.text.strip().split('\n')
-    metadata = dict(line.split(' ', 1) for line in metadata_raw[1:])
-    oid = metadata["oid"][7:] if metadata["oid"].startswith('sha256:') else metadata["oid"]
-    size = int(metadata['size'])
-
-    return oid, size
-
-def gitlfs_download_trace(repo_url, repo_commit, trace_path, oid, size):
-    headers = {
-        "Accept": "application/vnd.git-lfs+json",
-        "Content-Type": "application/vnd.git-lfs+json"
-    }
-    json = {
-        "operation": "download",
-        "transfers": [ "basic" ],
-        "ref": { "name": "refs/heads/%s" % repo_commit },
-        "objects": [
-            {
-                "oid": oid,
-                "size": size
-            }
-        ]
-    }
-
-    r = requests.post(repo_url + "/info/lfs/objects/batch", headers=headers, json=json)
-    url = r.json()["objects"][0]["actions"]["download"]["href"]
-    open(TRACES_DB_PATH + trace_path, "wb").write(requests.get(url).content)
-
-def checksum(filename, hash_factory=hashlib.sha256, chunk_num_blocks=128):
-    h = hash_factory()
-    with open(filename,'rb') as f:
-        for chunk in iter(lambda: f.read(chunk_num_blocks*h.block_size), b''):
-            h.update(chunk)
-    return h.hexdigest()
-
-def gitlab_ensure_trace(project_url, repo_commit, trace):
+def gitlab_ensure_trace(project_url, trace):
     trace_path = TRACES_DB_PATH + trace['path']
     if project_url is None:
-        assert(repo_commit is None)
         if not os.path.exists(trace_path):
             print("{} missing".format(trace_path))
             sys.exit(1)
@@ -93,18 +47,16 @@ def gitlab_ensure_trace(project_url, repo_commit, trace):
     os.makedirs(os.path.dirname(trace_path), exist_ok=True)
 
     if os.path.exists(trace_path):
-        local_oid = checksum(trace_path)
-
-    remote_oid, size = gitlab_download_metadata(project_url, repo_commit, trace['path'])
+        return
 
-    if not os.path.exists(trace_path) or local_oid != remote_oid:
-        print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True)
-        download_time = time.time()
-        gitlfs_download_trace(project_url + ".git", repo_commit, trace['path'], remote_oid, size)
-        print("took %ds." % (time.time() - download_time), flush=True)
+    print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True)
+    download_time = time.time()
+    r = requests.get(project_url + trace['path'])
+    open(trace_path, "wb").write(r.content)
+    print("took %ds." % (time.time() - download_time), flush=True)
 
-def gitlab_check_trace(project_url, repo_commit, device_name, trace, expectation):
-    gitlab_ensure_trace(project_url, repo_commit, trace)
+def gitlab_check_trace(project_url, device_name, trace, expectation):
+    gitlab_ensure_trace(project_url, trace)
 
     result = {}
     result[trace['path']] = {}
@@ -145,11 +97,9 @@ def run(filename, device_name):
         y = yaml.safe_load(f)
 
     if "traces-db" in y:
-        project_url = y["traces-db"]["gitlab-project-url"]
-        commit_id = y["traces-db"]["commit"]
+        project_url = y["traces-db"]["download-url"]
     else:
         project_url = None
-        commit_id = None
 
     traces = y['traces'] or []
     all_ok = True
@@ -157,7 +107,7 @@ def run(filename, device_name):
     for trace in traces:
         for expectation in trace['expectations']:
             if expectation['device'] == device_name:
-                ok, result = gitlab_check_trace(project_url, commit_id,
+                ok, result = gitlab_check_trace(project_url,
                                                 device_name, trace,
                                                 expectation)
                 all_ok = all_ok and ok