X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=debug%2Ftestlib.py;h=996c188efb8aff2a0c41e063cad437322b5361d9;hb=38fc7c6a96f47499b738835b0f01a42edf093d39;hp=f511088cf53e81c8fb9607db2c7aa3405e6d2b48;hpb=9063527a90e35c6d788823667d0280ad2f2db616;p=riscv-tests.git diff --git a/debug/testlib.py b/debug/testlib.py index f511088..996c188 100644 --- a/debug/testlib.py +++ b/debug/testlib.py @@ -17,8 +17,11 @@ import pexpect def find_file(path): for directory in (os.getcwd(), os.path.dirname(__file__)): fullpath = os.path.join(directory, path) - if os.path.exists(fullpath): - return fullpath + relpath = os.path.relpath(fullpath) + if len(relpath) >= len(fullpath): + relpath = fullpath + if os.path.exists(relpath): + return relpath return None def compile(args, xlen=32): # pylint: disable=redefined-builtin @@ -36,13 +39,12 @@ def compile(args, xlen=32): # pylint: disable=redefined-builtin cmd.append(found) else: cmd.append(arg) + header("Compile") + print "+", " ".join(cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode: - print - header("Compile failed") - print "+", " ".join(cmd) print stdout, print stderr, header("") @@ -63,26 +65,15 @@ class Spike(object): def __init__(self, target, halted=False, timeout=None, with_jtag_gdb=True): """Launch spike. Return tuple of its process and the port it's running on.""" - if target.sim_cmd: - cmd = shlex.split(target.sim_cmd) - else: - spike = os.path.expandvars("$RISCV/bin/spike") - cmd = [spike] - if target.xlen == 32: - cmd += ["--isa", "RV32G"] - else: - cmd += ["--isa", "RV64G"] - cmd += ["-m0x%x:0x%x" % (target.ram, target.ram_size)] + self.process = None - if timeout: - cmd = ["timeout", str(timeout)] + cmd + if target.harts: + harts = target.harts + else: + harts = [target] - if halted: - cmd.append('-H') - if with_jtag_gdb: - cmd += ['--rbb-port', '0'] - os.environ['REMOTE_BITBANG_HOST'] = 'localhost' - self.infinite_loop = target.compile( + cmd = self.command(target, harts, halted, timeout, with_jtag_gdb) + self.infinite_loop = target.compile(harts[0], "programs/checksum.c", "programs/tiny-malloc.c", "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE") cmd.append(self.infinite_loop) @@ -102,20 +93,60 @@ class Spike(object): os.environ['REMOTE_BITBANG_PORT'] = m.group(1) break time.sleep(0.11) - assert self.port, "Didn't get spike message about bitbang " \ - "connection" + if not self.port: + print_log(self.logname) + raise Exception("Didn't get spike message about bitbang " + "connection") + + def command(self, target, harts, halted, timeout, with_jtag_gdb): + # pylint: disable=no-self-use + if target.sim_cmd: + cmd = shlex.split(target.sim_cmd) + else: + spike = os.path.expandvars("$RISCV/bin/spike") + cmd = [spike] + + cmd += ["-p%d" % len(harts)] + + assert len(set(t.xlen for t in harts)) == 1, \ + "All spike harts must have the same XLEN" + + if harts[0].xlen == 32: + cmd += ["--isa", "RV32G"] + else: + cmd += ["--isa", "RV64G"] + + assert len(set(t.ram for t in harts)) == 1, \ + "All spike harts must have the same RAM layout" + assert len(set(t.ram_size for t in harts)) == 1, \ + "All spike harts must have the same RAM layout" + cmd += ["-m0x%x:0x%x" % (harts[0].ram, harts[0].ram_size)] + + if timeout: + cmd = ["timeout", str(timeout)] + cmd + + if halted: + cmd.append('-H') + if with_jtag_gdb: + cmd += ['--rbb-port', '0'] + os.environ['REMOTE_BITBANG_HOST'] = 'localhost' + + return cmd def __del__(self): - try: - self.process.kill() - self.process.wait() - except OSError: - pass + if self.process: + try: + self.process.kill() + self.process.wait() + except OSError: + pass def wait(self, *args, **kwargs): return self.process.wait(*args, **kwargs) class VcsSim(object): + logname = "simv.log" + def __init__(self, sim_cmd=None, debug=False): if sim_cmd: cmd = shlex.split(sim_cmd) @@ -125,10 +156,10 @@ class VcsSim(object): if debug: cmd[0] = cmd[0] + "-debug" cmd += ["+vcdplusfile=output/gdbserver.vpd"] - logfile = open("simv.log", "w") + logfile = open(self.logname, "w") logfile.write("+ %s\n" % " ".join(cmd)) logfile.flush() - listenfile = open("simv.log", "r") + listenfile = open(self.logname, "r") listenfile.seek(0, 2) self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=logfile, stderr=logfile) @@ -159,8 +190,11 @@ class VcsSim(object): class Openocd(object): logfile = tempfile.NamedTemporaryFile(prefix='openocd', suffix='.log') logname = logfile.name + print "OpenOCD Temporary Log File: %s" % logname + + def __init__(self, server_cmd=None, config=None, debug=False, timeout=60): + self.timeout = timeout - def __init__(self, server_cmd=None, config=None, debug=False): if server_cmd: cmd = shlex.split(server_cmd) else: @@ -197,66 +231,49 @@ class Openocd(object): logfile = open(Openocd.logname, "w") logfile.write("+ %s\n" % " ".join(cmd)) logfile.flush() - self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=logfile, stderr=logfile) - # Wait for OpenOCD to have made it through riscv_examine(). When using - # OpenOCD to communicate with a simulator this may take a long time, - # and gdb will time out when trying to connect if we attempt too early. - start = time.time() - messaged = False - while True: - log = open(Openocd.logname).read() - if "Ready for Remote Connections" in log: - break - if not self.process.poll() is None: - header("OpenOCD log") - sys.stdout.write(log) - raise Exception( - "OpenOCD exited before completing riscv_examine()") - if not messaged and time.time() - start > 1: - messaged = True - print "Waiting for OpenOCD to examine RISCV core..." - if time.time() - start > 60: - raise Exception("ERROR: Timed out waiting for OpenOCD to " - "examine RISCV core") + self.ports = [] + self.port = None + self.process = self.start(cmd, logfile) + + def start(self, cmd, logfile): + process = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=logfile, stderr=logfile) try: - self.port = self._get_gdb_server_port() - except: + # Wait for OpenOCD to have made it through riscv_examine(). When + # using OpenOCD to communicate with a simulator this may take a + # long time, and gdb will time out when trying to connect if we + # attempt too early. + start = time.time() + messaged = False + while True: + log = open(Openocd.logname).read() + m = re.search(r"Listening on port (\d+) for gdb connections", + log) + if m: + if not self.ports: + self.port = int(m.group(1)) + self.ports.append(int(m.group(1))) + + if "telnet server disabled" in log: + break + + if not process.poll() is None: + raise Exception( + "OpenOCD exited before completing riscv_examine()") + if not messaged and time.time() - start > 1: + messaged = True + print "Waiting for OpenOCD to start..." + if (time.time() - start) > self.timeout: + raise Exception("ERROR: Timed out waiting for OpenOCD to " + "listen for gdb") + return process + except Exception: header("OpenOCD log") sys.stdout.write(log) raise - def _get_gdb_server_port(self): - """Get port that OpenOCD's gdb server is listening on.""" - MAX_ATTEMPTS = 50 - PORT_REGEX = re.compile(r'(?P\d+) \(LISTEN\)') - for _ in range(MAX_ATTEMPTS): - with open(os.devnull, 'w') as devnull: - try: - output = subprocess.check_output([ - 'lsof', - '-a', # Take the AND of the following selectors - '-p{}'.format(self.process.pid), # Filter on PID - '-iTCP', # Filter only TCP sockets - ], stderr=devnull) - except subprocess.CalledProcessError: - output = "" - matches = list(PORT_REGEX.finditer(output)) - matches = [m for m in matches - if m.group('port') not in ('6666', '4444')] - if len(matches) > 1: - print output - raise Exception( - "OpenOCD listening on multiple ports. Cannot uniquely " - "identify gdb server port.") - elif matches: - [match] = matches - return int(match.group('port')) - time.sleep(1) - raise Exception("Timed out waiting for gdb server to obtain port.") - def __del__(self): try: self.process.kill() @@ -301,6 +318,7 @@ Thread = collections.namedtuple('Thread', ('id', 'target_id', 'name', class Gdb(object): logfile = tempfile.NamedTemporaryFile(prefix="gdb", suffix=".log") logname = logfile.name + print "GDB Temporary Log File: %s" % logname def __init__(self, cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb")): @@ -314,11 +332,16 @@ class Gdb(object): # Force consistency. self.command("set print entry-values no") + def select_hart(self, hart): + output = self.command("thread %d" % (hart.index + 1)) + assert "Unknown" not in output + def wait(self): """Wait for prompt.""" self.child.expect(r"\(gdb\)") def command(self, command, timeout=6000): + """timeout is in seconds""" self.child.sendline(command) self.child.expect("\n", timeout=timeout) self.child.expect(r"\(gdb\)", timeout=timeout) @@ -354,13 +377,23 @@ class Gdb(object): raise CannotAccess(int(m.group(1), 0)) return output.split('=')[-1].strip() - def p(self, obj): - output = self.command("p/x %s" % obj) + def parse_string(self, text): + text = text.strip() + if text.startswith("{") and text.endswith("}"): + inner = text[1:-1] + return [self.parse_string(t) for t in inner.split(", ")] + elif text.startswith('"') and text.endswith('"'): + return text[1:-1] + else: + return int(text, 0) + + def p(self, obj, fmt="/x"): + output = self.command("p%s %s" % (fmt, obj)) m = re.search("Cannot access memory at address (0x[0-9a-f]+)", output) if m: raise CannotAccess(int(m.group(1), 0)) - value = int(output.split('=')[-1].strip(), 0) - return value + rhs = output.split('=')[-1] + return self.parse_string(rhs) def p_string(self, obj): output = self.command("p %s" % obj) @@ -368,7 +401,7 @@ class Gdb(object): return value def stepi(self): - output = self.command("stepi") + output = self.command("stepi", timeout=60) return output def load(self): @@ -414,19 +447,20 @@ def run_all_tests(module, target, parsed): gdb_cmd = parsed.gdb todo = [] - if parsed.misaval: - target.misa = int(parsed.misaval, 16) - print "Using $misa from command line: 0x%x" % target.misa - elif target.misa: - print "Using $misa from target definition: 0x%x" % target.misa - else: - todo.append(("ExamineTarget", ExamineTarget)) + for hart in target.harts: + if parsed.misaval: + hart.misa = int(parsed.misaval, 16) + print "Using $misa from command line: 0x%x" % hart.misa + elif hart.misa: + print "Using $misa from hart definition: 0x%x" % hart.misa + else: + todo.append(("ExamineTarget", ExamineTarget, hart)) for name in dir(module): definition = getattr(module, name) if type(definition) == type and hasattr(definition, 'test') and \ (not parsed.test or any(test in name for test in parsed.test)): - todo.append((name, definition)) + todo.append((name, definition, None)) results, count = run_tests(parsed, target, todo) @@ -440,12 +474,12 @@ def run_tests(parsed, target, todo): results = {} count = 0 - for name, definition in todo: - instance = definition(target) + for name, definition, hart in todo: log_name = os.path.join(parsed.logs, "%s-%s-%s.log" % (time.strftime("%Y%m%d-%H%M%S"), type(target).__name__, name)) log_fd = open(log_name, 'w') - print "Running", name, "...", + print "Running %s > %s ..." % (name, log_name), + instance = definition(target, hart) sys.stdout.flush() log_fd.write("Test: %s\n" % name) log_fd.write("Target: %s\n" % type(target).__name__) @@ -459,8 +493,10 @@ def run_tests(parsed, target, todo): sys.stdout = real_stdout log_fd.write("Time elapsed: %.2fs\n" % (time.time() - start)) print "%s in %.2fs" % (result, time.time() - start) + if result not in good_results and parsed.print_failures: + sys.stdout.write(open(log_name).read()) sys.stdout.flush() - results.setdefault(result, []).append(name) + results.setdefault(result, []).append((name, log_name)) count += 1 if result not in good_results and parsed.fail_fast: break @@ -473,17 +509,18 @@ def print_results(results): print "%d tests returned %s" % (len(value), key) if key not in good_results: result = 1 - for test in value: - print " ", test + for name, log_name in value: + print " %s > %s" % (name, log_name) return result def add_test_run_options(parser): - parser.add_argument("--logs", default="logs", help="Store logs in the specified directory.") parser.add_argument("--fail-fast", "-f", action="store_true", help="Exit as soon as any test fails.") + parser.add_argument("--print-failures", action="store_true", + help="When a test fails, print the log file to stdout.") parser.add_argument("test", nargs='*', help="Run only tests that are named here.") parser.add_argument("--gdb", @@ -503,16 +540,20 @@ def header(title, dash='-', length=78): def print_log(path): header(path) - lines = open(path, "r").readlines() - for l in lines: + for l in open(path, "r"): sys.stdout.write(l) print class BaseTest(object): compiled = {} - def __init__(self, target): + def __init__(self, target, hart=None): self.target = target + if hart: + self.hart = hart + else: + self.hart = random.choice(target.harts) + self.hart = target.harts[-1] #<<< self.server = None self.target_process = None self.binary = None @@ -534,19 +575,29 @@ class BaseTest(object): if compile_args not in BaseTest.compiled: # pylint: disable=star-args BaseTest.compiled[compile_args] = \ - self.target.compile(*compile_args) + self.target.compile(self.hart, *compile_args) self.binary = BaseTest.compiled.get(compile_args) def classSetup(self): self.compile() self.target_process = self.target.create() - self.server = self.target.server() - self.logs.append(self.server.logname) + if self.target_process: + self.logs.append(self.target_process.logname) + try: + self.server = self.target.server() + self.logs.append(self.server.logname) + except Exception: + for log in self.logs: + print_log(log) + raise def classTeardown(self): del self.server del self.target_process + def postMortem(self): + pass + def run(self): """ If compile_args is set, compile a program and set self.binary. @@ -564,9 +615,8 @@ class BaseTest(object): self.start = time.time() - self.classSetup() - try: + self.classSetup() self.setup() result = self.test() # pylint: disable=no-member except TestNotApplicable: @@ -581,6 +631,7 @@ class BaseTest(object): print e.message header("Traceback") traceback.print_exc(file=sys.stdout) + self.postMortem() return result finally: @@ -595,8 +646,8 @@ class BaseTest(object): gdb_cmd = None class GdbTest(BaseTest): - def __init__(self, target): - BaseTest.__init__(self, target) + def __init__(self, target, hart=None): + BaseTest.__init__(self, target, hart=hart) self.gdb = None def classSetup(self): @@ -612,40 +663,57 @@ class GdbTest(BaseTest): if self.binary: self.gdb.command("file %s" % self.binary) if self.target: - self.gdb.command("set arch riscv:rv%d" % self.target.xlen) + self.gdb.command("set arch riscv:rv%d" % self.hart.xlen) self.gdb.command("set remotetimeout %d" % self.target.timeout_sec) if self.server.port: self.gdb.command( "target extended-remote localhost:%d" % self.server.port) - # Select a random thread. - # TODO: Allow a command line option to force a specific thread. - thread = random.choice(self.gdb.threads()) - self.gdb.thread(thread) + self.gdb.select_hart(self.hart) + + for cmd in self.target.gdb_setup: + self.gdb.command(cmd) # FIXME: OpenOCD doesn't handle PRIV now #self.gdb.p("$priv=3") + def postMortem(self): + if not self.gdb: + return + self.gdb.interrupt() + self.gdb.command("info registers all", timeout=10) + def classTeardown(self): del self.gdb BaseTest.classTeardown(self) +class GdbSingleHartTest(GdbTest): + def classSetup(self): + GdbTest.classSetup(self) + + for hart in self.target.harts: + # Park all harts that we're not using in a safe place. + if hart != self.hart: + self.gdb.select_hart(hart) + self.gdb.p("$pc=loop_forever") + self.gdb.select_hart(self.hart) + class ExamineTarget(GdbTest): def test(self): - self.target.misa = self.gdb.p("$misa") + self.hart.misa = self.gdb.p("$misa") txt = "RV" - if (self.target.misa >> 30) == 1: + if (self.hart.misa >> 30) == 1: txt += "32" - elif (self.target.misa >> 62) == 2: + elif (self.hart.misa >> 62) == 2: txt += "64" - elif (self.target.misa >> 126) == 3: + elif (self.hart.misa >> 126) == 3: txt += "128" else: raise TestFailed("Couldn't determine XLEN from $misa (0x%x)" % - self.target.misa) + self.hart.misa) for i in range(26): - if self.target.misa & (1<