tests,misc: Updated TestLib and boot-tests for gzipped imgs

In efforts to reduce storage costs and download times, the images hosted
by us have been gzipped. The TestLib framework has therefore been
extended to decompress gzipped files after download.

The x86-boot-tests are, at present, the only tests which use the gem5
images. These tests have been updated to download the gzipped image.

Change-Id: I6b2dbe9472a604148834820db8ea70e91e94376f
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/35257
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py
index bb911dde0ff28fe796ba3f1d30e8baa9dc4a9254..e21cb88598fcad89c12251d32f0ea70ae98e6689 100644 (file)
--- a/tests/gem5/fixture.py
+++ b/tests/gem5/fixture.py
@@ -42,6 +42,7 @@ import shutil
 import sys
 import socket
 import threading
+import gzip
 
 from six.moves import urllib
 
@@ -260,11 +261,11 @@ class DownloadedProgram(UniqueFixture):
         and downloads an updated version if it is needed.
     """
 
-    def __new__(cls, url, path, filename):
+    def __new__(cls, url, path, filename, gzip_decompress=False):
         target = joinpath(path, filename)
         return super(DownloadedProgram, cls).__new__(cls, target)
 
-    def _init(self, url, path, filename, **kwargs):
+    def _init(self, url, path, filename, gzip_decompress=False, **kwargs):
         """
         url: string
             The url of the archive
@@ -272,12 +273,16 @@ class DownloadedProgram(UniqueFixture):
             The absolute path of the directory containing the archive
         filename: string
             The name of the archive
+        gzip_decompress: boolean
+            True if this target resource have been compressed using gzip and
+            is to be decompressed prior to usage.
         """
 
         self.url = url
         self.path = path
         self.filename = joinpath(path, filename)
         self.name = "Downloaded:" + self.filename
+        self.gzip_decompress = gzip_decompress
 
     def _download(self):
         import errno
@@ -288,7 +293,17 @@ class DownloadedProgram(UniqueFixture):
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
-        urllib.request.urlretrieve(self.url, self.filename)
+        if self.gzip_decompress:
+            gzipped_filename = self.filename + ".gz"
+            urllib.request.urlretrieve(self.url, gzipped_filename)
+
+            with open(self.filename, 'w') as outfile:
+                with gzip.open(gzipped_filename, 'r') as infile:
+                    shutil.copyfileobj(infile, outfile)
+
+            os.remove(gzipped_filename)
+        else:
+            urllib.request.urlretrieve(self.url, self.filename)
 
     def _getremotetime(self):
         import datetime, time

diff --git a/tests/gem5/x86-boot-tests/test_linux_boot.py b/tests/gem5/x86-boot-tests/test_linux_boot.py
index 0e66659202bd59cfa41c48e80d8bd0daee883f9c..d73f3a1202a5c4882588143a79341eec29666103 100644 (file)
--- a/tests/gem5/x86-boot-tests/test_linux_boot.py
+++ b/tests/gem5/x86-boot-tests/test_linux_boot.py
@@ -34,13 +34,13 @@ else:
     base_path = joinpath(absdirpath(__file__), '..', 'resources',
             'ubuntu-boot')
 
-image_url = config.resource_url + '/images/x86/ubuntu-18-04/base.img'
+image_url = config.resource_url + '/images/x86/ubuntu-18-04/base.img.gz'
 kernel_url = config.resource_url + '/kernels/x86/static/vmlinux-4.19.83'
 
 image_name = 'ubuntu-18-04-base.img'
 kernel_name = 'vmlinux-4.19.83' # 4.19 is LTS (Projected EOL: Dec, 2020)
 
-image = DownloadedProgram(image_url, base_path, image_name)
+image = DownloadedProgram(image_url, base_path, image_name, True)
 kernel = DownloadedProgram(kernel_url, base_path, kernel_name)