From 20507f8b17f87c19f821270c0a893ca9acc9dbee Mon Sep 17 00:00:00 2001 From: Tomeu Vizoso Date: Thu, 9 Jul 2020 12:42:02 +0200 Subject: [PATCH] gitlab-ci: Download traces from MinIO Downloading the traces directly from git causes very high egress from GCE, which is expensive. So we can expand trace testing further, we are going to keep a cache in freedesktop.org's MinIO instance. This commit implements downloading from it. Signed-off-by: Tomeu Vizoso Reviewed-by: Daniel Stone Reviewed-By: Rohan Garg Part-of: --- .gitlab-ci/traces.yml | 3 +- .gitlab-ci/tracie/tracie.py | 72 ++++++------------------------------- 2 files changed, 12 insertions(+), 63 deletions(-) diff --git a/.gitlab-ci/traces.yml b/.gitlab-ci/traces.yml index 1ec7603407f..87db9e94856 100644 --- a/.gitlab-ci/traces.yml +++ b/.gitlab-ci/traces.yml @@ -1,6 +1,5 @@ traces-db: - gitlab-project-url: "https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db" - commit: "d3b1efe0cd69ef6ae40a29a14ed733ee0ba0cb4c" + download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" traces: - path: glmark2/desktop-blur-radius=5:effect=blur:passes=1:separable=true:windows=4.rdc diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py index 59016cae164..b09bf4a9014 100644 --- a/.gitlab-ci/tracie/tracie.py +++ b/.gitlab-ci/tracie/tracie.py @@ -36,55 +36,9 @@ def replay(trace_path, device_name): log_file = files[0] return hashlib.md5(Image.open(image_file).tobytes()).hexdigest(), image_file, log_file -def gitlab_download_metadata(project_url, repo_commit, trace_path): - url = parse.urlparse(project_url) - - url_path = url.path - if url_path.startswith("/"): - url_path = url_path[1:] - - gitlab_api_url = url.scheme + "://" + url.netloc + "/api/v4/projects/" + parse.quote_plus(url_path) - - r = requests.get(gitlab_api_url + "/repository/files/%s/raw?ref=%s" % (parse.quote_plus(trace_path), repo_commit)) - metadata_raw = r.text.strip().split('\n') - metadata = dict(line.split(' ', 1) for line in metadata_raw[1:]) - oid = metadata["oid"][7:] if metadata["oid"].startswith('sha256:') else metadata["oid"] - size = int(metadata['size']) - - return oid, size - -def gitlfs_download_trace(repo_url, repo_commit, trace_path, oid, size): - headers = { - "Accept": "application/vnd.git-lfs+json", - "Content-Type": "application/vnd.git-lfs+json" - } - json = { - "operation": "download", - "transfers": [ "basic" ], - "ref": { "name": "refs/heads/%s" % repo_commit }, - "objects": [ - { - "oid": oid, - "size": size - } - ] - } - - r = requests.post(repo_url + "/info/lfs/objects/batch", headers=headers, json=json) - url = r.json()["objects"][0]["actions"]["download"]["href"] - open(TRACES_DB_PATH + trace_path, "wb").write(requests.get(url).content) - -def checksum(filename, hash_factory=hashlib.sha256, chunk_num_blocks=128): - h = hash_factory() - with open(filename,'rb') as f: - for chunk in iter(lambda: f.read(chunk_num_blocks*h.block_size), b''): - h.update(chunk) - return h.hexdigest() - -def gitlab_ensure_trace(project_url, repo_commit, trace): +def gitlab_ensure_trace(project_url, trace): trace_path = TRACES_DB_PATH + trace['path'] if project_url is None: - assert(repo_commit is None) if not os.path.exists(trace_path): print("{} missing".format(trace_path)) sys.exit(1) @@ -93,18 +47,16 @@ def gitlab_ensure_trace(project_url, repo_commit, trace): os.makedirs(os.path.dirname(trace_path), exist_ok=True) if os.path.exists(trace_path): - local_oid = checksum(trace_path) - - remote_oid, size = gitlab_download_metadata(project_url, repo_commit, trace['path']) + return - if not os.path.exists(trace_path) or local_oid != remote_oid: - print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True) - download_time = time.time() - gitlfs_download_trace(project_url + ".git", repo_commit, trace['path'], remote_oid, size) - print("took %ds." % (time.time() - download_time), flush=True) + print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True) + download_time = time.time() + r = requests.get(project_url + trace['path']) + open(trace_path, "wb").write(r.content) + print("took %ds." % (time.time() - download_time), flush=True) -def gitlab_check_trace(project_url, repo_commit, device_name, trace, expectation): - gitlab_ensure_trace(project_url, repo_commit, trace) +def gitlab_check_trace(project_url, device_name, trace, expectation): + gitlab_ensure_trace(project_url, trace) result = {} result[trace['path']] = {} @@ -145,11 +97,9 @@ def run(filename, device_name): y = yaml.safe_load(f) if "traces-db" in y: - project_url = y["traces-db"]["gitlab-project-url"] - commit_id = y["traces-db"]["commit"] + project_url = y["traces-db"]["download-url"] else: project_url = None - commit_id = None traces = y['traces'] or [] all_ok = True @@ -157,7 +107,7 @@ def run(filename, device_name): for trace in traces: for expectation in trace['expectations']: if expectation['device'] == device_name: - ok, result = gitlab_check_trace(project_url, commit_id, + ok, result = gitlab_check_trace(project_url, device_name, trace, expectation) all_ok = all_ok and ok -- 2.30.2