From c79dcaf498b546f10583b91f4f9745c24036c4d4 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Fri, 25 Sep 2020 10:25:43 -0700 Subject: [PATCH] tests,misc: Updated TestLib and boot-tests for gzipped imgs In efforts to reduce storage costs and download times, the images hosted by us have been gzipped. The TestLib framework has therefore been extended to decompress gzipped files after download. The x86-boot-tests are, at present, the only tests which use the gem5 images. These tests have been updated to download the gzipped image. Change-Id: I6b2dbe9472a604148834820db8ea70e91e94376f Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/35257 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Tested-by: kokoro --- tests/gem5/fixture.py | 21 +++++++++++++++++--- tests/gem5/x86-boot-tests/test_linux_boot.py | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py index bb911dde0..e21cb8859 100644 --- a/tests/gem5/fixture.py +++ b/tests/gem5/fixture.py @@ -42,6 +42,7 @@ import shutil import sys import socket import threading +import gzip from six.moves import urllib @@ -260,11 +261,11 @@ class DownloadedProgram(UniqueFixture): and downloads an updated version if it is needed. """ - def __new__(cls, url, path, filename): + def __new__(cls, url, path, filename, gzip_decompress=False): target = joinpath(path, filename) return super(DownloadedProgram, cls).__new__(cls, target) - def _init(self, url, path, filename, **kwargs): + def _init(self, url, path, filename, gzip_decompress=False, **kwargs): """ url: string The url of the archive @@ -272,12 +273,16 @@ class DownloadedProgram(UniqueFixture): The absolute path of the directory containing the archive filename: string The name of the archive + gzip_decompress: boolean + True if this target resource have been compressed using gzip and + is to be decompressed prior to usage. """ self.url = url self.path = path self.filename = joinpath(path, filename) self.name = "Downloaded:" + self.filename + self.gzip_decompress = gzip_decompress def _download(self): import errno @@ -288,7 +293,17 @@ class DownloadedProgram(UniqueFixture): except OSError as e: if e.errno != errno.EEXIST: raise - urllib.request.urlretrieve(self.url, self.filename) + if self.gzip_decompress: + gzipped_filename = self.filename + ".gz" + urllib.request.urlretrieve(self.url, gzipped_filename) + + with open(self.filename, 'w') as outfile: + with gzip.open(gzipped_filename, 'r') as infile: + shutil.copyfileobj(infile, outfile) + + os.remove(gzipped_filename) + else: + urllib.request.urlretrieve(self.url, self.filename) def _getremotetime(self): import datetime, time diff --git a/tests/gem5/x86-boot-tests/test_linux_boot.py b/tests/gem5/x86-boot-tests/test_linux_boot.py index 0e6665920..d73f3a120 100644 --- a/tests/gem5/x86-boot-tests/test_linux_boot.py +++ b/tests/gem5/x86-boot-tests/test_linux_boot.py @@ -34,13 +34,13 @@ else: base_path = joinpath(absdirpath(__file__), '..', 'resources', 'ubuntu-boot') -image_url = config.resource_url + '/images/x86/ubuntu-18-04/base.img' +image_url = config.resource_url + '/images/x86/ubuntu-18-04/base.img.gz' kernel_url = config.resource_url + '/kernels/x86/static/vmlinux-4.19.83' image_name = 'ubuntu-18-04-base.img' kernel_name = 'vmlinux-4.19.83' # 4.19 is LTS (Projected EOL: Dec, 2020) -image = DownloadedProgram(image_url, base_path, image_name) +image = DownloadedProgram(image_url, base_path, image_name, True) kernel = DownloadedProgram(kernel_url, base_path, kernel_name) -- 2.30.2