From 93933b44654e5e1f94cdbdc3efed84aa4a642b82 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 21 Apr 2020 02:22:14 -0700 Subject: [PATCH] misc,tests: Updated and generalized the resource url for tests As we are now going to maintain different bundles of resources for each gem5 release, the resources have been archived to http://dist.gem5.org/dist/current for gem5 19. The development branch will use http://dist.gem5.org/dist/develop going forward. New releases will follow the format http://dist.gem5.org/dist/{VERSION}. This patch makes the resources url a command-line parameter, set to the "correct" url by default. This will be updated to the correct, archived, version subdirectory upon release of a new gem5 version. E.g.: http://dist.gem5.org/dist/v20 for the gem5 20 release. Some Make files have been cleaned up to no longer fetch and push to remote locations. As gem5-resources is implemented, sources will be moved to the gem5-resources resository and compiled binaries, etc. pulled from our Google Cloud bucket from the tests themselves. Change-Id: Ia16c496be3a60283ecc431ffaa5b059e1932b526 Issue-on: https://gem5.atlassian.net/browse/GEM5-431 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/27987 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Tested-by: kokoro --- ext/testlib/config.py | 7 +++++++ tests/gem5/cpu_tests/test.py | 2 +- tests/gem5/fs/linux/arm/test.py | 2 +- tests/gem5/hello_se/test_hello_se.py | 2 +- tests/gem5/insttest_se/test.py | 2 +- tests/gem5/m5_util/test_exit.py | 3 +-- tests/gem5/m5threads_test_atomic/test.py | 2 +- tests/gem5/x86-boot-tests/test_linux_boot.py | 4 ++-- tests/test-progs/hello/src/Makefile.arm | 8 -------- tests/test-progs/hello/src/Makefile.x86 | 10 ---------- tests/test-progs/m5-exit/src/Makefile.x86 | 9 --------- 11 files changed, 15 insertions(+), 36 deletions(-) diff --git a/ext/testlib/config.py b/ext/testlib/config.py index 39d6a24ce..348d0a84c 100644 --- a/ext/testlib/config.py +++ b/ext/testlib/config.py @@ -220,6 +220,7 @@ def define_defaults(defaults): os.pardir)) defaults.result_path = os.path.join(os.getcwd(), '.testing-results') defaults.list_only_failed = False + defaults.resource_url = 'http://dist.gem5.org/dist/develop' def define_constants(constants): ''' @@ -579,6 +580,12 @@ def define_common_args(config): default=None, help='Path where binaries are stored (downloaded if not present)' ), + Argument( + '--resource-url', + action='store', + default=config._defaults.resource_url, + help='The URL where the resources reside.' + ), ] diff --git a/tests/gem5/cpu_tests/test.py b/tests/gem5/cpu_tests/test.py index ba90e2a90..339d15aae 100644 --- a/tests/gem5/cpu_tests/test.py +++ b/tests/gem5/cpu_tests/test.py @@ -56,7 +56,7 @@ if config.bin_path: else: base_path = joinpath(absdirpath(__file__), 'benchmarks', 'bin') -base_url = 'http://dist.gem5.org/dist/current/gem5/cpu_tests/benchmarks/bin/' +base_url = config.resource_url + '/gem5/cpu_tests/benchmarks/bin/' for isa in valid_isas: path = joinpath(base_path, isa) for workload in workloads: diff --git a/tests/gem5/fs/linux/arm/test.py b/tests/gem5/fs/linux/arm/test.py index f90299bf3..39cb01135 100644 --- a/tests/gem5/fs/linux/arm/test.py +++ b/tests/gem5/fs/linux/arm/test.py @@ -81,7 +81,7 @@ arm_fs_long_tests = [ ] tarball = 'aarch-system-201901106.tar.bz2' -url = "http://dist.gem5.org/dist/current/arm/" + tarball +url = config.resource_url + "/arm/" + tarball filepath = os.path.dirname(os.path.abspath(__file__)) path = config.bin_path if config.bin_path else filepath arm_fs_binaries = DownloadedArchive(url, path, tarball) diff --git a/tests/gem5/hello_se/test_hello_se.py b/tests/gem5/hello_se/test_hello_se.py index 75043b91f..260daad51 100644 --- a/tests/gem5/hello_se/test_hello_se.py +++ b/tests/gem5/hello_se/test_hello_se.py @@ -81,7 +81,7 @@ else: base_path = joinpath(absdirpath(__file__), '..', 'test-progs', 'hello', 'bin') -urlbase = 'http://dist.gem5.org/dist/current/test-progs/hello/bin/' +urlbase = config.resource_url + '/test-progs/hello/bin/' ref_path = joinpath(getcwd(), 'ref') verifiers = ( diff --git a/tests/gem5/insttest_se/test.py b/tests/gem5/insttest_se/test.py index c3fa3225c..6bddc8616 100644 --- a/tests/gem5/insttest_se/test.py +++ b/tests/gem5/insttest_se/test.py @@ -51,7 +51,7 @@ if config.bin_path: else: base_path = joinpath(absdirpath(__file__), '..', 'test-progs') -urlbase = 'http://dist.gem5.org/dist/current/test-progs/insttest/bin/' +urlbase = config.resource_url + '/test-progs/insttest/bin/' for isa in test_progs: for binary in test_progs[isa]: for operating_s in supported_os[isa]: diff --git a/tests/gem5/m5_util/test_exit.py b/tests/gem5/m5_util/test_exit.py index 02e97cfa6..0a0dc1621 100644 --- a/tests/gem5/m5_util/test_exit.py +++ b/tests/gem5/m5_util/test_exit.py @@ -53,8 +53,7 @@ else: path = joinpath(absdirpath(__file__), '..', 'test-progs', 'hello', 'bin', 'x86', 'linux') filename = 'm5_exit' -url = ('http://dist.gem5.org/dist/current/test-progs/' + - 'm5-exit/bin/x86/linux/m5_exit') +url = (config.resource_url + '/test-progs/m5-exit/bin/x86/linux/m5_exit') test_program = DownloadedProgram(url, path, filename) a = verifier.MatchRegex(m5_exit_regex) diff --git a/tests/gem5/m5threads_test_atomic/test.py b/tests/gem5/m5threads_test_atomic/test.py index 64af6364c..b8e1e47ff 100644 --- a/tests/gem5/m5threads_test_atomic/test.py +++ b/tests/gem5/m5threads_test_atomic/test.py @@ -38,7 +38,7 @@ else: 'test_atomic', 'bin') binary = 'test_atomic' -url = 'http://dist.gem5.org/dist/current/test-progs/pthread/bin/' + binary +url = config.resource_url + '/current/test-progs/pthread/bin/' + binary DownloadedProgram(url, base_path, binary) verifiers = ( diff --git a/tests/gem5/x86-boot-tests/test_linux_boot.py b/tests/gem5/x86-boot-tests/test_linux_boot.py index 0417eb9b7..0e6665920 100644 --- a/tests/gem5/x86-boot-tests/test_linux_boot.py +++ b/tests/gem5/x86-boot-tests/test_linux_boot.py @@ -34,8 +34,8 @@ else: base_path = joinpath(absdirpath(__file__), '..', 'resources', 'ubuntu-boot') -image_url = 'http://dist.gem5.org/images/x86/ubuntu-18-04/base.img' -kernel_url = 'http://dist.gem5.org/kernels/x86/static/vmlinux-4.19.83' +image_url = config.resource_url + '/images/x86/ubuntu-18-04/base.img' +kernel_url = config.resource_url + '/kernels/x86/static/vmlinux-4.19.83' image_name = 'ubuntu-18-04-base.img' kernel_name = 'vmlinux-4.19.83' # 4.19 is LTS (Projected EOL: Dec, 2020) diff --git a/tests/test-progs/hello/src/Makefile.arm b/tests/test-progs/hello/src/Makefile.arm index 1f022469d..4a18483bd 100644 --- a/tests/test-progs/hello/src/Makefile.arm +++ b/tests/test-progs/hello/src/Makefile.arm @@ -1,13 +1,5 @@ -SERVER_USER= -SERVER_PATH=/z/www/htdocs/dist/current/test-progs/hello/bin/arm/linux -FETCH_PATH=http://dist.gem5.org/dist/current/test-progs/hello/bin/arm/linux -UPLOAD_LOCATION=$(SERVER_USER)daystrom.gem5.org:$(SERVER_PATH) - all: hello64-static hello32-static -upload: hello64-static hello32-static - scp hello64-static hello32-static $(UPLOAD_LOCATION) - hello64-static: hello.c dockcross-arm64 ./dockcross-arm64 bash -c '$$CC hello.c -o hello64-static -static' diff --git a/tests/test-progs/hello/src/Makefile.x86 b/tests/test-progs/hello/src/Makefile.x86 index 3df2d23c1..70d89b046 100644 --- a/tests/test-progs/hello/src/Makefile.x86 +++ b/tests/test-progs/hello/src/Makefile.x86 @@ -1,15 +1,5 @@ -SERVER_USER= -SERVER_PATH=/z/www/htdocs/dist/current/test-progs/hello/bin/x86/linux -FETCH_PATH=http://dist.gem5.org/dist/current/test-progs/hello/bin/x86/linux - -UPLOAD_LOCATION=$(SERVER_USER)daystrom.gem5.org:$(SERVER_PATH) - all: hello64-static hello64-dynamic hello32-static - -upload: hello64-static hello64-dynamic hello32-static - scp hello64-static hello64-dynamic hello32-static $(UPLOAD_LOCATION) - hello64-static: hello.c dockcross-x64 ./dockcross-x64 bash -c '$$CC hello.c -o hello64-static -static' diff --git a/tests/test-progs/m5-exit/src/Makefile.x86 b/tests/test-progs/m5-exit/src/Makefile.x86 index 2c94a54c9..4ab1b875e 100644 --- a/tests/test-progs/m5-exit/src/Makefile.x86 +++ b/tests/test-progs/m5-exit/src/Makefile.x86 @@ -1,17 +1,8 @@ -SERVER_USER= -SERVER_PATH=/z/www/htdocs/dist/current/test-progs/m5-exit/bin/x86/linux -FETCH_PATH=http://dist.gem5.org/dist/current/test-progs/m5-exit/bin/x86/linux - -UPLOAD_LOCATION=$(SERVER_USER)daystrom.gem5.org:$(SERVER_PATH) - M5_UTIL:=${CURDIR}/../../../../util/m5 M5_INCLUDE:=${CURDIR}/../../../../include/ all: m5_exit -upload: m5_exit - scp m5_exit $(UPLOAD_LOCATION) - # NOTE: For docker to work, all of the files must be in this directory m5_exit: m5-exit.c cp -r $(M5_INCLUDE)/gem5 . -- 2.30.2