From 7ffd684334e93fb0268e114802e55e8e118694ee Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Thu, 16 Apr 2020 11:55:17 -0700 Subject: [PATCH] tests,python: Upgrading testlib to function with Python2 Change-Id: I9926b1507e9069ae8564c31bdd377b2b916462a2 Issue-on: https://gem5.atlassian.net/browse/GEM5-395 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/29088 Reviewed-by: Bobby R. Bruce Maintainer: Bobby R. Bruce Tested-by: kokoro --- ext/testlib/__init__.py | 6 +- ext/testlib/{config.py => configuration.py} | 8 +-- ext/testlib/fixture.py | 6 +- ext/testlib/handlers.py | 16 +++--- ext/testlib/helper.py | 6 +- ext/testlib/loader.py | 47 ++++++++------- ext/testlib/log.py | 5 +- ext/testlib/main.py | 64 ++++++++++----------- ext/testlib/query.py | 4 +- ext/testlib/result.py | 18 +++--- ext/testlib/runner.py | 12 ++-- ext/testlib/sandbox.py | 10 ++-- ext/testlib/suite.py | 6 +- ext/testlib/terminal.py | 29 +++++----- ext/testlib/{test.py => test_util.py} | 6 +- ext/testlib/uid.py | 6 +- ext/testlib/wrappers.py | 7 ++- tests/configs/base_config.py | 4 +- tests/gem5/__init__.py | 8 +-- tests/gem5/cpu_tests/test.py | 4 +- tests/gem5/fixture.py | 17 +++--- tests/gem5/memory/test.py | 4 +- tests/gem5/suite.py | 12 ++-- tests/gem5/verifier.py | 4 +- tests/main.py | 6 +- tests/testing/__init__.py | 2 +- tests/testing/helpers.py | 2 +- tests/testing/results.py | 5 +- tests/testing/tests.py | 17 ++++-- tests/testing/units.py | 11 ++-- tests/tests.py | 2 +- 31 files changed, 190 insertions(+), 164 deletions(-) rename ext/testlib/{config.py => configuration.py} (99%) rename ext/testlib/{test.py => test_util.py} (97%) diff --git a/ext/testlib/__init__.py b/ext/testlib/__init__.py index 893da5433..898205d9a 100644 --- a/ext/testlib/__init__.py +++ b/ext/testlib/__init__.py @@ -29,12 +29,12 @@ from .state import * from .runner import * -from .test import * +from .test_util import * from .suite import * from .loader import * from .fixture import * -from .config import * -from main import main +from .configuration import * +from .main import main #TODO Remove this awkward bootstrap #FIXME diff --git a/ext/testlib/config.py b/ext/testlib/configuration.py similarity index 99% rename from ext/testlib/config.py rename to ext/testlib/configuration.py index 189f7c1d5..04744e113 100644 --- a/ext/testlib/config.py +++ b/ext/testlib/configuration.py @@ -83,10 +83,11 @@ import copy import os import re -from ConfigParser import ConfigParser +from six import add_metaclass +from six.moves import configparser as ConfigParser from pickle import HIGHEST_PROTOCOL as highest_pickle_protocol -from helper import absdirpath, AttrDict, FrozenAttrDict +from testlib.helper import absdirpath, AttrDict, FrozenAttrDict class UninitialzedAttributeException(Exception): ''' @@ -598,9 +599,8 @@ def define_common_args(config): # one in the list will be saved. common_args = AttrDict({arg.name:arg for arg in common_args}) - +@add_metaclass(abc.ABCMeta) class ArgParser(object): - __metaclass__ = abc.ABCMeta def __init__(self, parser): # Copy public methods of the parser. diff --git a/ext/testlib/fixture.py b/ext/testlib/fixture.py index 7af6cb289..20568b80f 100644 --- a/ext/testlib/fixture.py +++ b/ext/testlib/fixture.py @@ -29,8 +29,8 @@ import copy import traceback -import helper -import log +import testlib.helper as helper +import testlib.log as log class SkipException(Exception): def __init__(self, fixture, testitem): @@ -61,7 +61,7 @@ class Fixture(object): collector = helper.InstanceCollector() def __new__(klass, *args, **kwargs): - obj = super(Fixture, klass).__new__(klass, *args, **kwargs) + obj = super(Fixture, klass).__new__(klass) Fixture.collector.collect(obj) return obj diff --git a/ext/testlib/handlers.py b/ext/testlib/handlers.py index 6f7694071..3005e01ef 100644 --- a/ext/testlib/handlers.py +++ b/ext/testlib/handlers.py @@ -35,20 +35,20 @@ from __future__ import print_function import multiprocessing import os -import Queue import sys import threading import time import traceback -import helper -import log -import result -import state -import test -import terminal +import testlib.helper as helper +import testlib.log as log +import testlib.result as result +import testlib.state as state +import testlib.test_util as test +import testlib.terminal as terminal -from config import config, constants +from six.moves import queue as Queue +from testlib.configuration import config, constants class _TestStreamManager(object): diff --git a/ext/testlib/helper.py b/ext/testlib/helper.py index ac49e468e..4237a765e 100644 --- a/ext/testlib/helper.py +++ b/ext/testlib/helper.py @@ -34,7 +34,6 @@ from collections import MutableSet, OrderedDict import difflib import errno import os -import Queue import re import shutil import stat @@ -44,6 +43,8 @@ import threading import time import traceback +from six.moves import queue as Queue + #TODO Tear out duplicate logic from the sandbox IOManager def log_call(logger, command, *popenargs, **kwargs): ''' @@ -80,7 +81,8 @@ def log_call(logger, command, *popenargs, **kwargs): def log_output(log_callback, pipe, redirects=tuple()): # Read iteractively, don't allow input to fill the pipe. - for line in iter(pipe.readline, ''): + for line in iter(pipe.readline, b''): + line = line.decode("utf-8") for r in redirects: r.write(line) log_callback(line.rstrip()) diff --git a/ext/testlib/loader.py b/ext/testlib/loader.py index 8f8f60e70..bb2fe4ef9 100644 --- a/ext/testlib/loader.py +++ b/ext/testlib/loader.py @@ -67,16 +67,17 @@ a :class:`TestSuite` named after the module. import os import re +import six import sys import traceback -import config -import log -import suite as suite_mod -import test as test_mod -import fixture as fixture_mod -import wrappers -import uid +import testlib.configuration as configuration +import testlib.log as log +import testlib.suite as suite_mod +import testlib.test_util as test_mod +import testlib.fixture as fixture_mod +import testlib.wrappers as wrappers +import testlib.uid as uid class DuplicateTestItemException(Exception): ''' @@ -112,7 +113,7 @@ def _assert_files_in_same_dir(files): if files: directory = os.path.dirname(files[0]) for f in files: - assert os.path.dirname(f) == directory + assert(os.path.dirname(f) == directory) class Loader(object): ''' @@ -186,6 +187,7 @@ class Loader(object): self._loaded_a_file = True for directory in self._discover_files(root): + directory = list(directory) if directory: _assert_files_in_same_dir(directory) for f in directory: @@ -193,6 +195,7 @@ class Loader(object): def load_dir(self, directory): for dir_ in self._discover_files(directory): + directory = list(directory) _assert_files_in_same_dir(dir_) for f in dir_: self.load_file(f) @@ -221,29 +224,27 @@ class Loader(object): sys.path.insert(0, os.path.dirname(path)) cwd = os.getcwd() os.chdir(os.path.dirname(path)) - config.config.file_under_load = path + configuration.config.file_under_load = path new_tests = test_mod.TestCase.collector.create() new_suites = suite_mod.TestSuite.collector.create() new_fixtures = fixture_mod.Fixture.collector.create() - def cleanup(): - config.config.file_under_load = None - sys.path[:] = old_path - os.chdir(cwd) - test_mod.TestCase.collector.remove(new_tests) - suite_mod.TestSuite.collector.remove(new_suites) - fixture_mod.Fixture.collector.remove(new_fixtures) - try: - execfile(path, newdict, newdict) + exec(open(path).read(), newdict, newdict) except Exception as e: log.test_log.debug(traceback.format_exc()) log.test_log.warn( 'Exception thrown while loading "%s"\n' 'Ignoring all tests in this file.' % (path)) - cleanup() + # Clean up + configuration.config.file_under_load = None + sys.path[:] = old_path + os.chdir(cwd) + test_mod.TestCase.collector.remove(new_tests) + suite_mod.TestSuite.collector.remove(new_suites) + fixture_mod.Fixture.collector.remove(new_fixtures) return # Create a module test suite for those not contained in a suite. @@ -281,7 +282,13 @@ class Loader(object): self.suites.extend(loaded_suites) self.suite_uids.update({suite.uid: suite for suite in loaded_suites}) - cleanup() + # Clean up + configuration.config.file_under_load = None + sys.path[:] = old_path + os.chdir(cwd) + test_mod.TestCase.collector.remove(new_tests) + suite_mod.TestSuite.collector.remove(new_suites) + fixture_mod.Fixture.collector.remove(new_fixtures) def _discover_files(self, root): ''' diff --git a/ext/testlib/log.py b/ext/testlib/log.py index 5ba6f5d4f..cddb9217c 100644 --- a/ext/testlib/log.py +++ b/ext/testlib/log.py @@ -30,8 +30,9 @@ This module supplies the global `test_log` object which all testing results and messages are reported through. ''' -import wrappers +import testlib.wrappers as wrappers +from six import add_metaclass class LogLevel(): Fatal = 0 @@ -55,6 +56,7 @@ class RecordTypeCounterMetaclass(type): RecordTypeCounterMetaclass.counter += 1 +@add_metaclass(RecordTypeCounterMetaclass) class Record(object): ''' A generic object that is passed to the :class:`Log` and its handlers. @@ -62,7 +64,6 @@ class Record(object): ..note: Although not statically enforced, all items in the record should be be pickleable. This enables logging accross multiple processes. ''' - __metaclass__ = RecordTypeCounterMetaclass def __init__(self, **data): self.data = data diff --git a/ext/testlib/main.py b/ext/testlib/main.py index 3827f7815..9c9ed03dd 100644 --- a/ext/testlib/main.py +++ b/ext/testlib/main.py @@ -29,16 +29,16 @@ import os import itertools -import config -import fixture as fixture_mod -import handlers -import loader as loader_mod -import log -import query -import result -import runner -import terminal -import uid +import testlib.configuration as configuration +import testlib.fixture as fixture_mod +import testlib.handlers as handlers +import testlib.loader as loader_mod +import testlib.log as log +import testlib.query as query +import testlib.result as result +import testlib.runner as runner +import testlib.terminal as terminal +import testlib.uid as uid def entry_message(): log.test_log.message("Running the new gem5 testing script.") @@ -50,7 +50,7 @@ def entry_message(): class RunLogHandler(): def __init__(self): term_handler = handlers.TerminalHandler( - verbosity=config.config.verbose+log.LogLevel.Info + verbosity=configuration.config.verbose+log.LogLevel.Info ) summary_handler = handlers.SummaryHandler() self.mp_handler = handlers.MultiprocessingHandlerWrapper( @@ -62,7 +62,7 @@ class RunLogHandler(): def schedule_finalized(self, test_schedule): # Create the result handler object. self.result_handler = handlers.ResultHandler( - test_schedule, config.config.result_path) + test_schedule, configuration.config.result_path) self.mp_handler.add_handler(self.result_handler) def finish_testing(self): @@ -87,27 +87,27 @@ class RunLogHandler(): return self.result_handler.unsuccessful() def get_config_tags(): - return getattr(config.config, - config.StorePositionalTagsAction.position_kword) + return getattr(configuration.config, + configuration.StorePositionalTagsAction.position_kword) def filter_with_config_tags(loaded_library): tags = get_config_tags() final_tags = [] regex_fmt = '^%s$' - cfg = config.config + cfg = configuration.config def _append_inc_tag_filter(name): if hasattr(cfg, name): tag_opts = getattr(cfg, name) for tag in tag_opts: - final_tags.append(config.TagRegex(True, regex_fmt % tag)) + final_tags.append(configuration.TagRegex(True, regex_fmt % tag)) def _append_rem_tag_filter(name): if hasattr(cfg, name): tag_opts = getattr(cfg, name) for tag in cfg.constants.supported_tags[name]: if tag not in tag_opts: - final_tags.append(config.TagRegex(False, regex_fmt % tag)) + final_tags.append(configuration.TagRegex(False, regex_fmt % tag)) # Append additional tags for the isa, length, and variant options. # They apply last (they take priority) @@ -206,13 +206,13 @@ def load_tests(): testloader = loader_mod.Loader() log.test_log.message(terminal.separator()) log.test_log.message('Loading Tests', bold=True) - testloader.load_root(config.config.directory) + testloader.load_root(configuration.config.directory) return testloader def do_list(): term_handler = handlers.TerminalHandler( - verbosity=config.config.verbose+log.LogLevel.Info, - machine_only=config.config.quiet + verbosity=configuration.config.verbose+log.LogLevel.Info, + machine_only=configuration.config.quiet ) log.test_log.log_obj.add_handler(term_handler) @@ -223,11 +223,11 @@ def do_list(): qrunner = query.QueryRunner(test_schedule) - if config.config.suites: + if configuration.config.suites: qrunner.list_suites() - elif config.config.tests: + elif configuration.config.tests: qrunner.list_tests() - elif config.config.all_tags: + elif configuration.config.all_tags: qrunner.list_tags() else: qrunner.list_suites() @@ -259,13 +259,13 @@ def run_schedule(test_schedule, log_handler): log.test_log.message('Running Tests from {} suites' .format(len(test_schedule.suites)), bold=True) log.test_log.message("Results will be stored in {}".format( - config.config.result_path)) + configuration.config.result_path)) log.test_log.message(terminal.separator()) # Build global fixtures and exectute scheduled test suites. - if config.config.test_threads > 1: + if configuration.config.test_threads > 1: library_runner = runner.LibraryParallelRunner(test_schedule) - library_runner.set_threads(config.config.test_threads) + library_runner.set_threads(configuration.config.test_threads) else: library_runner = runner.LibraryRunner(test_schedule) library_runner.run() @@ -279,8 +279,8 @@ def run_schedule(test_schedule, log_handler): def do_run(): # Initialize early parts of the log. with RunLogHandler() as log_handler: - if config.config.uid: - uid_ = uid.UID.from_uid(config.config.uid) + if configuration.config.uid: + uid_ = uid.UID.from_uid(configuration.config.uid) if isinstance(uid_, uid.TestUID): log.test_log.error('Unable to run a standalone test.\n' 'Gem5 expects test suites to be the smallest unit ' @@ -305,8 +305,8 @@ def do_rerun(): with RunLogHandler() as log_handler: # Load previous results results = result.InternalSavedResults.load( - os.path.join(config.config.result_path, - config.constants.pickle_filename)) + os.path.join(configuration.config.result_path, + configuration.constants.pickle_filename)) rerun_suites = (suite.uid for suite in results if suite.unsuccessful) @@ -323,10 +323,10 @@ def main(): Returns 0 on success and 1 otherwise so it can be used as a return code for scripts. ''' - config.initialize_config() + configuration.initialize_config() # 'do' the given command. - result = globals()['do_'+config.config.command]() + result = globals()['do_'+configuration.config.command]() log.test_log.close() return result diff --git a/ext/testlib/query.py b/ext/testlib/query.py index c66445c44..174af626f 100644 --- a/ext/testlib/query.py +++ b/ext/testlib/query.py @@ -26,8 +26,8 @@ # # Authors: Sean Wilson -import terminal -import log +import testlib.terminal as terminal +import testlib.log as log # TODO Refactor print logic out of this so the objects # created are separate from print logic. diff --git a/ext/testlib/result.py b/ext/testlib/result.py index 786c21b60..38b3322ba 100644 --- a/ext/testlib/result.py +++ b/ext/testlib/result.py @@ -30,10 +30,10 @@ import os import pickle import xml.sax.saxutils -from config import config -import helper -import state -import log +from testlib.configuration import config +import testlib.helper as helper +import testlib.state as state +import testlib.log as log def _create_uid_index(iterable): index = {} @@ -62,7 +62,7 @@ class _CommonMetadataMixin: return self._metadata.result.value != state.Result.Passed -class InternalTestResult(object, _CommonMetadataMixin): +class InternalTestResult(_CommonMetadataMixin): def __init__(self, obj, suite, directory): self._metadata = obj.metadata self.suite = suite @@ -77,7 +77,7 @@ class InternalTestResult(object, _CommonMetadataMixin): ) -class InternalSuiteResult(object, _CommonMetadataMixin): +class InternalSuiteResult(_CommonMetadataMixin): def __init__(self, obj, directory): self._metadata = obj.metadata self.directory = directory @@ -104,7 +104,7 @@ class InternalSuiteResult(object, _CommonMetadataMixin): return results -class InternalLibraryResults(object, _CommonMetadataMixin): +class InternalLibraryResults(_CommonMetadataMixin): def __init__(self, obj, directory): self.directory = directory self._metadata = obj.metadata @@ -159,12 +159,12 @@ class InternalSavedResults: if exc.errno != errno.EEXIST: raise - with open(path, 'w') as f: + with open(path, 'wb') as f: pickle.dump(results, f, protocol) @staticmethod def load(path): - with open(path, 'r') as f: + with open(path, 'rb') as f: return pickle.load(f) diff --git a/ext/testlib/runner.py b/ext/testlib/runner.py index 9868cefb1..ec3c838e7 100644 --- a/ext/testlib/runner.py +++ b/ext/testlib/runner.py @@ -30,13 +30,13 @@ import multiprocessing.dummy import threading import traceback -import helper -import state -import log -import sandbox +import testlib.helper as helper +import testlib.state as state +import testlib.log as log +import testlib.sandbox as sandbox -from state import Status, Result -from fixture import SkipException +from testlib.state import Status, Result +from testlib.fixture import SkipException def compute_aggregate_result(iterable): ''' diff --git a/ext/testlib/sandbox.py b/ext/testlib/sandbox.py index 49fe133ea..bdc6d8859 100644 --- a/ext/testlib/sandbox.py +++ b/ext/testlib/sandbox.py @@ -33,7 +33,7 @@ import sys import threading import traceback -import log +import testlib.log as log pdb._Pdb = pdb.Pdb class ForkedPdb(pdb._Pdb): @@ -81,18 +81,18 @@ class IoManager(object): self.old_stdout = os.dup(sys.stdout.fileno()) os.dup2(self.stderr_wp, sys.stderr.fileno()) - sys.stderr = os.fdopen(self.stderr_wp, 'w', 0) + sys.stderr = os.fdopen(self.stderr_wp, 'w') os.dup2(self.stdout_wp, sys.stdout.fileno()) - sys.stdout = os.fdopen(self.stdout_wp, 'w', 0) + sys.stdout = os.fdopen(self.stdout_wp, 'w') def restore_pipes(self): self.stderr_wp = os.dup(sys.stderr.fileno()) self.stdout_wp = os.dup(sys.stdout.fileno()) os.dup2(self.old_stderr, sys.stderr.fileno()) - sys.stderr = os.fdopen(self.old_stderr, 'w', 0) + sys.stderr = open(self.old_stderr, 'w') os.dup2(self.old_stdout, sys.stdout.fileno()) - sys.stdout = os.fdopen(self.old_stdout, 'w', 0) + sys.stdout = open(self.old_stdout, 'w') def start_loggers(self): self.log_ouput() diff --git a/ext/testlib/suite.py b/ext/testlib/suite.py index 2ce817a6e..eae52fd92 100644 --- a/ext/testlib/suite.py +++ b/ext/testlib/suite.py @@ -27,8 +27,8 @@ # Authors: Sean Wilson -import helper -import runner as runner_mod +import testlib.helper as helper +import testlib.runner as runner_mod class TestSuite(object): ''' @@ -52,7 +52,7 @@ class TestSuite(object): tags = set() def __new__(klass, *args, **kwargs): - obj = super(TestSuite, klass).__new__(klass, *args, **kwargs) + obj = super(TestSuite, klass).__new__(klass) TestSuite.collector.collect(obj) return obj diff --git a/ext/testlib/terminal.py b/ext/testlib/terminal.py index bdb20edea..bc4c85599 100644 --- a/ext/testlib/terminal.py +++ b/ext/testlib/terminal.py @@ -28,6 +28,7 @@ import sys import fcntl import termios import struct +import six # Intended usage example: # @@ -74,7 +75,7 @@ try: def cap_string(s, *args): cap = curses.tigetstr(s) if cap: - return curses.tparm(cap, *args) + return curses.tparm(cap, *args).decode("utf-8") else: return '' except: @@ -84,7 +85,7 @@ class ColorStrings(object): def __init__(self, cap_string): for i, c in enumerate(color_names): setattr(self, c, cap_string('setaf', i)) - for name, cap in capability_map.iteritems(): + for name, cap in six.iteritems(capability_map): setattr(self, name, cap_string(cap)) termcap = ColorStrings(cap_string) @@ -137,7 +138,7 @@ def insert_separator(inside, char=default_separator, .. seealso:: :func:`separator` ''' # Use a bytearray so it's efficient to manipulate - string = bytearray(separator(char, color=color)) + string = bytearray(separator(char, color=color), 'utf-8') # Check if we can fit inside with at least min_barrier. gap = (len(string) - len(inside)) - min_barrier * 2 @@ -145,27 +146,27 @@ def insert_separator(inside, char=default_separator, # We'll need to expand the string to fit us. string.extend([ char for _ in range(-gap)]) # Emplace inside - middle = ((len(string)-1)/2) - start_idx = middle - len(inside)/2 - string[start_idx:len(inside)+start_idx] = inside - return str(string) + middle = (len(string)-1)//2 + start_idx = middle - len(inside)//2 + string[start_idx:len(inside)+start_idx] = str.encode(inside) + return str(string.decode("utf-8")) if __name__ == '__main__': def test_termcap(obj): for c_name in color_names: c_str = getattr(obj, c_name) - print c_str + c_name + obj.Normal + print(c_str + c_name + obj.Normal) for attr_name in capability_names: if attr_name == 'Normal': continue attr_str = getattr(obj, attr_name) - print attr_str + c_str + attr_name + " " + c_name + obj.Normal - print obj.Bold + obj.Underline + \ - c_name + "Bold Underline " + c_str + obj.Normal + print(attr_str + c_str + attr_name + " " + c_name + obj.Normal) + print(obj.Bold + obj.Underline + \ + c_name + "Bold Underline " + c_str + obj.Normal) - print "=== termcap enabled ===" + print("=== termcap enabled ===") test_termcap(termcap) - print termcap.Normal - print "=== termcap disabled ===" + print(termcap.Normal) + print("=== termcap disabled ===") test_termcap(no_termcap) diff --git a/ext/testlib/test.py b/ext/testlib/test_util.py similarity index 97% rename from ext/testlib/test.py rename to ext/testlib/test_util.py index 18899d6d0..5a0c0a8f1 100644 --- a/ext/testlib/test.py +++ b/ext/testlib/test_util.py @@ -28,8 +28,8 @@ import functools -import helper -import runner as runner_mod +import testlib.helper as helper +import testlib.runner as runner_mod class TestingException(Exception): '''Common ancestor for manual Testing Exceptions.''' @@ -62,7 +62,7 @@ class TestCase(object): collector = helper.InstanceCollector() def __new__(cls, *args, **kwargs): - obj = super(TestCase, cls).__new__(cls, *args, **kwargs) + obj = super(TestCase, cls).__new__(cls) TestCase.collector.collect(obj) return obj diff --git a/ext/testlib/uid.py b/ext/testlib/uid.py index fe56252a0..f8951a28d 100644 --- a/ext/testlib/uid.py +++ b/ext/testlib/uid.py @@ -29,7 +29,7 @@ import os import itertools -import config +import testlib.configuration as configuration class UID(object): sep = ':' @@ -42,12 +42,12 @@ class UID(object): @staticmethod def _shorten_path(path): return os.path.relpath(path, - os.path.commonprefix((config.constants.testing_base, + os.path.commonprefix((configuration.constants.testing_base, path))) @staticmethod def _full_path(short_path): - return os.path.join(config.constants.testing_base, short_path) + return os.path.join(configuration.constants.testing_base, short_path) @classmethod def uid_to_path(cls, uid): diff --git a/ext/testlib/wrappers.py b/ext/testlib/wrappers.py index 4bd22a468..e91970262 100644 --- a/ext/testlib/wrappers.py +++ b/ext/testlib/wrappers.py @@ -44,9 +44,8 @@ loaded by the testlib :class:`testlib.loader.Loader`. ''' import itertools -import log -import uid -from state import Status, Result +import testlib.uid as uid +from testlib.state import Status, Result class TestCaseMetadata(): def __init__(self, name, uid, path, result, status, suite_uid): @@ -127,9 +126,11 @@ class LoadedTestable(object): # TODO Change log to provide status_update, result_update for all types. def log_status(self, status): + import testlib.log as log log.test_log.status_update(self, status) def log_result(self, result): + import testlib.log as log log.test_log.result_update(self, result) def __iter__(self): diff --git a/tests/configs/base_config.py b/tests/configs/base_config.py index 0f7993875..b124a132b 100644 --- a/tests/configs/base_config.py +++ b/tests/configs/base_config.py @@ -43,9 +43,11 @@ from common import FSConfig from common import Options from common.Caches import * from ruby import Ruby +from six import add_metaclass _have_kvm_support = 'BaseKvmCPU' in globals() +@add_metaclass(ABCMeta) class BaseSystem(object): """Base system builder. @@ -55,8 +57,6 @@ class BaseSystem(object): the initialization process. """ - __metaclass__ = ABCMeta - def __init__(self, mem_mode='timing', mem_class=SimpleMemory, cpu_class=TimingSimpleCPU, num_cpus=1, num_threads=1, checker=False, mem_size=None, use_ruby=False): diff --git a/tests/gem5/__init__.py b/tests/gem5/__init__.py index 614165a48..0955469d0 100644 --- a/tests/gem5/__init__.py +++ b/tests/gem5/__init__.py @@ -24,8 +24,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import suite -import fixture +import testlib.suite +import testlib.fixture -from suite import * -from fixture import * +from .suite import * +from .fixture import * diff --git a/tests/gem5/cpu_tests/test.py b/tests/gem5/cpu_tests/test.py index 339d15aae..a21c4b965 100644 --- a/tests/gem5/cpu_tests/test.py +++ b/tests/gem5/cpu_tests/test.py @@ -70,11 +70,11 @@ for isa in valid_isas: binary = joinpath(workload_binary.path, workload) for cpu in valid_isas[isa]: - gem5_verify_config( + gem5_verify_config( name='cpu_test_{}_{}'.format(cpu,workload), verifiers=verifiers, config=joinpath(getcwd(), 'run.py'), config_args=['--cpu={}'.format(cpu), binary], valid_isas=(isa.upper(),), fixtures=[workload_binary] - ) + ) diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py index fc31b30c6..f28201b04 100644 --- a/tests/gem5/fixture.py +++ b/tests/gem5/fixture.py @@ -42,11 +42,11 @@ import shutil import sys import socket import threading -import urllib -import urllib2 + +from six.moves import urllib from testlib.fixture import Fixture -from testlib.config import config, constants +from testlib.configuration import config, constants from testlib.helper import log_call, cacheresult, joinpath, absdirpath import testlib.log as log from testlib.state import Result @@ -271,15 +271,16 @@ class DownloadedProgram(UniqueFixture): except OSError as e: if e.errno != errno.EEXIST: raise - urllib.urlretrieve(self.url, self.filename) + urllib.request.urlretrieve(self.url, self.filename) def _getremotetime(self): import datetime, time import _strptime # Needed for python threading bug - u = urllib2.urlopen(self.url, timeout=10) + u = urllib.request.urlopen(self.url, timeout=10) + return time.mktime(datetime.datetime.strptime( \ - u.info().getheaders("Last-Modified")[0], + u.info()["Last-Modified"], "%a, %d %b %Y %X GMT").timetuple()) def _setup(self, testitem): @@ -289,7 +290,7 @@ class DownloadedProgram(UniqueFixture): else: try: t = self._getremotetime() - except (urllib2.URLError, socket.timeout): + except (urllib.error.URLError, socket.timeout): # Problem checking the server, use the old files. log.test_log.debug("Could not contact server. Binaries may be old.") return @@ -315,7 +316,7 @@ class DownloadedArchive(DownloadedProgram): else: try: t = self._getremotetime() - except (urllib2.URLError, socket.timeout): + except (urllib.error.URLError, socket.timeout): # Problem checking the server, use the old files. log.test_log.debug("Could not contact server. " "Binaries may be old.") diff --git a/tests/gem5/memory/test.py b/tests/gem5/memory/test.py index 2a4eeb36f..bf87a278c 100644 --- a/tests/gem5/memory/test.py +++ b/tests/gem5/memory/test.py @@ -28,6 +28,8 @@ Test file for simple memory test TODO: Add stats checking ''' +import six + from testlib import * gem5_verify_config( @@ -48,7 +50,7 @@ simple_mem_params = [ for name, params in simple_mem_params: - args = ['--' + key + '=' + val for key,val in params.iteritems()] + args = ['--' + key + '=' + val for key,val in six.iteritems(params)] gem5_verify_config( name='simple_mem_' + name, diff --git a/tests/gem5/suite.py b/tests/gem5/suite.py index 25e652e84..4cf0f81a2 100644 --- a/tests/gem5/suite.py +++ b/tests/gem5/suite.py @@ -41,12 +41,13 @@ import copy import subprocess import sys -from testlib.test import TestFunction +from testlib.test_util import TestFunction from testlib.suite import TestSuite from testlib.helper import log_call -from testlib.config import constants, config -from fixture import TempdirFixture, Gem5Fixture, VariableFixture -import verifier +from testlib.configuration import constants, config +from .fixture import TempdirFixture, Gem5Fixture, VariableFixture + +from . import verifier def gem5_verify_config(name, config, @@ -190,6 +191,7 @@ def _create_test_run_gem5(config, config_args, gem5_args): command.append(config) # Config_args should set up the program args. command.extend(config_args) - returncode.value = log_call(params.log, command, stderr=sys.stderr) + returncode.value = log_call(params.log, command, stdout=sys.stdout, + stderr=sys.stderr) return test_run_gem5 diff --git a/tests/gem5/verifier.py b/tests/gem5/verifier.py index c955c407d..815b9bbaf 100644 --- a/tests/gem5/verifier.py +++ b/tests/gem5/verifier.py @@ -29,8 +29,8 @@ Built in test cases that verify particular details about a gem5 run. ''' import re -from testlib import test -from testlib.config import constants +from testlib import test_util as test +from testlib.configuration import constants from testlib.helper import joinpath, diff_out_file class Verifier(object): diff --git a/tests/main.py b/tests/main.py index 5cd68e91d..3287ef14c 100755 --- a/tests/main.py +++ b/tests/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python ''' The main source for testlib. Ties together the default test runners and loaders. @@ -10,6 +10,8 @@ from __future__ import print_function import sys import os +os.environ["PYTHONUNBUFFERED"] = "1" + base_dir = os.path.dirname(os.path.abspath(__file__)) ext_path = os.path.join(base_dir, os.pardir, 'ext') @@ -17,7 +19,7 @@ sys.path.insert(0, base_dir) sys.path.insert(0, ext_path) import testlib.main as testlib -import testlib.config as config +import testlib.configuration as config import testlib.helper as helper config.basedir = helper.absdirpath(__file__) diff --git a/tests/testing/__init__.py b/tests/testing/__init__.py index 4f97291d5..5dcc684da 100644 --- a/tests/testing/__init__.py +++ b/tests/testing/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # # Copyright (c) 2016 ARM Limited # All rights reserved diff --git a/tests/testing/helpers.py b/tests/testing/helpers.py index 6fc1e28a0..f46915c8f 100755 --- a/tests/testing/helpers.py +++ b/tests/testing/helpers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # # Copyright (c) 2016 ARM Limited # All rights reserved diff --git a/tests/testing/results.py b/tests/testing/results.py index 42ec24524..eb88d4c23 100644 --- a/tests/testing/results.py +++ b/tests/testing/results.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # # Copyright (c) 2016 ARM Limited # All rights reserved @@ -40,6 +40,7 @@ from __future__ import print_function from abc import ABCMeta, abstractmethod import inspect import pickle +from six import add_metaclass import string import sys @@ -141,8 +142,8 @@ class TestResult(object): def __nonzero__(self): return all([ r for r in self.results ]) +@add_metaclass(ABCMeta) class ResultFormatter(object): - __metaclass__ = ABCMeta def __init__(self, fout=sys.stdout, verbose=False): self.verbose = verbose diff --git a/tests/testing/tests.py b/tests/testing/tests.py index 29bfa78b6..042180d79 100755 --- a/tests/testing/tests.py +++ b/tests/testing/tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # # Copyright (c) 2016-2017 ARM Limited # All rights reserved @@ -38,9 +38,15 @@ from abc import ABCMeta, abstractmethod import os from collections import namedtuple -from .units import * -from .helpers import FileIgnoreList -from .results import TestResult + +from six import add_metaclass + +import sys +sys.path.append(os.path.dirname(__file__)) + +from units import * +from helpers import FileIgnoreList +from results import TestResult import shutil _test_base = os.path.join(os.path.dirname(__file__), "..") @@ -172,6 +178,7 @@ def get_default_protocol(arch): all_categories = ("quick", "long") all_modes = ("fs", "se") +@add_metaclass(ABCMeta) class Test(object): """Test case base class. @@ -182,8 +189,6 @@ class Test(object): """ - __metaclass__ = ABCMeta - def __init__(self, name): self.test_name = name diff --git a/tests/testing/units.py b/tests/testing/units.py index 9c9c1e5bf..d1fa6b227 100644 --- a/tests/testing/units.py +++ b/tests/testing/units.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # # Copyright (c) 2016 ARM Limited # All rights reserved @@ -41,15 +41,18 @@ import difflib import functools import os import re +from six import add_metaclass import subprocess import sys import traceback -from .results import UnitResult -from .helpers import * +sys.path.append(os.path.dirname(__file__)) +from results import UnitResult +from helpers import * _test_base = os.path.join(os.path.dirname(__file__), "..") +@add_metaclass(ABCMeta) class TestUnit(object): """Base class for all test units. @@ -64,8 +67,6 @@ class TestUnit(object): """ - __metaclass__ = ABCMeta - def __init__(self, name, ref_dir, test_dir, skip=False): self.name = name self.ref_dir = ref_dir diff --git a/tests/tests.py b/tests/tests.py index b23cb5a1d..df46b414c 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # # Copyright (c) 2016 ARM Limited # All rights reserved -- 2.30.2