tests/gem5/fixture.py

   1 # Copyright (c) 2017 Mark D. Hill and David A. Wood
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met: redistributions of source code must retain the above copyright
   7 # notice, this list of conditions and the following disclaimer;
   8 # redistributions in binary form must reproduce the above copyright
   9 # notice, this list of conditions and the following disclaimer in the
  10 # documentation and/or other materials provided with the distribution;
  11 # neither the name of the copyright holders nor the names of its
  12 # contributors may be used to endorse or promote products derived from
  13 # this software without specific prior written permission.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  16 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  17 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  18 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  19 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  20 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  21 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 #
  27 # Authors: Sean Wilson
  28
  29 import os
  30 import tempfile
  31 import shutil
  32
  33 from testlib.fixture import Fixture, globalfixture
  34 from testlib.config import config, constants
  35 from testlib.helper import log_call, cacheresult, joinpath, absdirpath
  36 import testlib.log as log
  37
  38
  39 class VariableFixture(Fixture):
  40     def __init__(self, value=None, name=None):
  41         super(VariableFixture, self).__init__(name=name)
  42         self.value = value
  43
  44
  45 class TempdirFixture(Fixture):
  46     def __init__(self):
  47         self.path = None
  48         super(TempdirFixture, self).__init__(
  49                 name=constants.tempdir_fixture_name)
  50
  51     def setup(self, testitem):
  52         self.path = tempfile.mkdtemp(prefix='gem5out')
  53
  54     def teardown(self, testitem):
  55         if self.path is not None:
  56             shutil.rmtree(self.path)
  57
  58     def skip_cleanup(self):
  59         # Set path to none so it's not deleted
  60         self.path = None
  61
  62
  63 class SConsFixture(Fixture):
  64     '''
  65     Fixture will wait until all SCons targets are collected and tests are
  66     about to be ran, then will invocate a single instance of SCons for all
  67     targets.
  68
  69     :param directory: The directory which scons will -C (cd) into before
  70         executing. If None is provided, will choose the config base_dir.
  71     '''
  72     def __init__(self, directory=None, target_class=None):
  73         self.directory = directory if directory else config.base_dir
  74         self.target_class = target_class if target_class else SConsTarget
  75         self.threads = config.threads
  76         self.targets = set()
  77         super(SConsFixture, self).__init__()
  78
  79     def setup(self, testitem):
  80         if config.skip_build:
  81             return
  82
  83         command = [
  84             'scons', '-C', self.directory,
  85             '-j', str(self.threads),
  86             '--ignore-style'
  87         ]
  88
  89         if not self.targets:
  90             log.test_log.warn(
  91                 'No SCons targets specified, this will'
  92                 ' build the default all target.\n'
  93                 'This is likely unintended, and you'
  94                 ' may wish to kill testlib and reconfigure.')
  95         else:
  96             log.test_log.message(
  97                     'Building the following targets.'
  98                     ' This may take a while.')
  99             log.test_log.message('%s' % (', '.join(self.targets)))
 100             log.test_log.message(
 101                     "You may want to run with only a single ISA"
 102                     "(--isa=), use --skip-build, or use 'rerun'.")
 103
 104
 105
 106         command.extend(self.targets)
 107         log_call(log.test_log, command)
 108
 109
 110 class SConsTarget(Fixture):
 111     # The singleton scons fixture we'll use for all targets.
 112     default_scons_invocation = None
 113
 114     def __init__(self, target, build_dir=None, invocation=None):
 115         '''
 116         Represents a target to be built by an 'invocation' of scons.
 117
 118         :param target: The target known to scons.
 119
 120         :param build_dir: The 'build' directory path which will be prepended
 121             to the target name.
 122
 123         :param invocation: Represents an invocation of scons which we will
 124             automatically attach this target to. If None provided, uses the
 125             main 'scons' invocation.
 126         '''
 127
 128         if build_dir is None:
 129             build_dir = config.build_dir
 130         self.target = os.path.join(build_dir, target)
 131         super(SConsTarget, self).__init__(name=target)
 132
 133         if invocation is None:
 134             if self.default_scons_invocation is None:
 135                 SConsTarget.default_scons_invocation = SConsFixture()
 136                 globalfixture(SConsTarget.default_scons_invocation)
 137
 138             invocation = self.default_scons_invocation
 139         self.invocation = invocation
 140
 141     def schedule_finalized(self, schedule):
 142         self.invocation.targets.add(self.target)
 143         return Fixture.schedule_finalized(self, schedule)
 144
 145 class Gem5Fixture(SConsTarget):
 146     def __init__(self, isa, variant):
 147         target = joinpath(isa.upper(), 'gem5.%s' % variant)
 148         super(Gem5Fixture, self).__init__(target)
 149
 150         self.name = constants.gem5_binary_fixture_name
 151         self.path = self.target
 152         self.isa = isa
 153         self.variant = variant
 154
 155
 156 class MakeFixture(Fixture):
 157     def __init__(self, directory, *args, **kwargs):
 158         name = 'make -C %s' % directory
 159         super(MakeFixture, self).__init__(build_once=True, lazy_init=False,
 160                                           name=name,
 161                                           *args, **kwargs)
 162         self.targets = []
 163         self.directory = directory
 164
 165     def setup(self):
 166         super(MakeFixture, self).setup()
 167         targets = set(self.required_by)
 168         command = ['make', '-C', self.directory]
 169         command.extend([target.target for target in targets])
 170         log_call(command)
 171
 172
 173 class MakeTarget(Fixture):
 174     def __init__(self, target, make_fixture=None, *args, **kwargs):
 175         '''
 176         :param make_fixture: The make invocation we will be attached to.
 177         Since we don't have a single global instance of make in gem5 like we do
 178         scons we need to know what invocation to attach to. If none given,
 179         creates its own.
 180         '''
 181         super(MakeTarget, self).__init__(name=target, *args, **kwargs)
 182         self.target = self.name
 183
 184         if make_fixture is None:
 185             make_fixture = MakeFixture(
 186                     absdirpath(target),
 187                     lazy_init=True,
 188                     build_once=False)
 189
 190         self.make_fixture = make_fixture
 191
 192         # Add our self to the required targets of the main MakeFixture
 193         self.require(self.make_fixture)
 194
 195     def setup(self, testitem):
 196         super(MakeTarget, self).setup()
 197         self.make_fixture.setup()
 198         return self
 199
 200 class TestProgram(MakeTarget):
 201     def __init__(self, program, isa, os, recompile=False):
 202         make_dir = joinpath('test-progs', program)
 203         make_fixture = MakeFixture(make_dir)
 204         target = joinpath('bin', isa, os, program)
 205         super(TestProgram, self).__init__(target, make_fixture)
 206         self.path = joinpath(make_dir, target)
 207         self.recompile = recompile
 208
 209     def setup(self, testitem):
 210         # Check if the program exists if it does then only compile if
 211         # recompile was given.
 212         if self.recompile:
 213             super(MakeTarget, self).setup()
 214         elif not os.path.exists(self.path):
 215             super(MakeTarget, self).setup()
 216
 217 class DownloadedProgram(Fixture):
 218     """ Like TestProgram, but checks the version in the gem5 binary repository
 219         and downloads an updated version if it is needed.
 220     """
 221     urlbase = "http://gem5.org/dist/current/"
 222
 223     def __init__(self, path, program, **kwargs):
 224         """
 225         path: string
 226             The path to the directory containing the binary relative to
 227             $GEM5_BASE/tests
 228         program: string
 229             The name of the binary file
 230         """
 231         super(DownloadedProgram, self).__init__("download-" + program,
 232                                                 build_once=True, **kwargs)
 233
 234         self.program_dir = path
 235         relative_path = joinpath(self.program_dir, program)
 236         self.url = self.urlbase + relative_path
 237         self.path = os.path.realpath(
 238                         joinpath(absdirpath(__file__), '../', relative_path)
 239                     )
 240
 241     def _download(self):
 242         import urllib
 243         import errno
 244         log.test_log.debug("Downloading " + self.url + " to " + self.path)
 245         if not os.path.exists(self.program_dir):
 246             try:
 247                 os.makedirs(self.program_dir)
 248             except OSError as e:
 249                 if e.errno != errno.EEXIST:
 250                     raise
 251         urllib.urlretrieve(self.url, self.path)
 252
 253     def _getremotetime(self):
 254         import  urllib2, datetime, time
 255         import _strptime # Needed for python threading bug
 256
 257         u = urllib2.urlopen(self.url)
 258         return time.mktime(datetime.datetime.strptime( \
 259                     u.info().getheaders("Last-Modified")[0],
 260                     "%a, %d %b %Y %X GMT").timetuple())
 261
 262     def setup(self, testitem):
 263         import urllib2
 264         # Check to see if there is a file downloaded
 265         if not os.path.exists(self.path):
 266             self._download()
 267         else:
 268             try:
 269                 t = self._getremotetime()
 270             except urllib2.URLError:
 271                 # Problem checking the server, use the old files.
 272                 log.debug("Could not contact server. Binaries may be old.")
 273                 return
 274             # If the server version is more recent, download it
 275             if t > os.path.getmtime(self.path):
 276                 self._download()