Fix tests to work in multi-gdb mode.
authorTim Newsome <tim@sifive.com>
Fri, 29 Sep 2017 20:20:30 +0000 (13:20 -0700)
committerTim Newsome <tim@sifive.com>
Fri, 29 Sep 2017 20:20:30 +0000 (13:20 -0700)
The Gdb class now can handle connecting to more than one gdb. It
enumerates the harts across all connections, and when asked to select a
hart, it transparently sends future gdb commands to the correct
instance.

Multicore tests still have to be aware of some differences. The main one
is that when executing 'c' in RTOS mode, all harts resume, while in
multi-gdb mode only the current one resumes. Additionally, gdb doesn't
set breakpoints until 'c' is issued, so the hart where breakpoints are
set needs to be resumed before other harts might see them.

14 files changed:
debug/Makefile
debug/gdbserver.py
debug/targets.py
debug/targets/RISC-V/spike-1.cfg [new file with mode: 0644]
debug/targets/RISC-V/spike-2.cfg [new file with mode: 0644]
debug/targets/RISC-V/spike-rtos.cfg
debug/targets/RISC-V/spike.cfg [deleted file]
debug/targets/RISC-V/spike32-2-rtos.py [new file with mode: 0644]
debug/targets/RISC-V/spike32-2.py
debug/targets/RISC-V/spike32.py
debug/targets/RISC-V/spike64-2-rtos.py [new file with mode: 0644]
debug/targets/RISC-V/spike64-2.py
debug/targets/RISC-V/spike64.py
debug/testlib.py

index 33988dd02615fae0a40517bedc85f80f7b4a29cf..9f7cb2ed19845bdcd08809fa062007dc0dd6310b 100644 (file)
@@ -6,7 +6,7 @@ GDBSERVER_PY = $(src_dir)/gdbserver.py
 
 default: spike$(XLEN)-2
 
-all:   pylint spike32 spike64 spike32-2 spike64-2
+all:   pylint spike32 spike32-2 spike32-2-rtos spike64 spike64-2 spike64-2-rtos
 
 pylint:
        pylint --rcfile=pylint.rc `git ls-files '*.py'`
index 135dab82a5ea6c952f385e8482fabebc41d0bf26..924f42ae69e0fb23fc5d181f92a9719570b1a082 100755 (executable)
@@ -363,7 +363,7 @@ class Hwbp2(DebugTest):
         self.exit()
 
 class TooManyHwbp(DebugTest):
-    def run(self):
+    def test(self):
         for i in range(30):
             self.gdb.hbreak("*rot13 + %d" % (i * 4))
 
@@ -476,21 +476,27 @@ class MulticoreRegTest(GdbTest):
 
     def test(self):
         # Run to main
+        # Hart 0 is the first to be resumed, so we have to set the breakpoint
+        # there. gdb won't actually set the breakpoint until we tell it to
+        # resume.
+        self.gdb.select_hart(self.target.harts[0])
         self.gdb.b("main")
-        self.gdb.c()
-        for t in self.gdb.threads():
-            assertIn("main", t.frame)
+        self.gdb.c_all()
+        for hart in self.target.harts:
+            self.gdb.select_hart(hart)
+            assertIn("main", self.gdb.where())
+        self.gdb.select_hart(self.target.harts[0])
         self.gdb.command("delete breakpoints")
 
         # Run through the entire loop.
         self.gdb.b("main_end")
-        self.gdb.c()
+        self.gdb.c_all()
 
         hart_ids = []
-        for t in self.gdb.threads():
-            assertIn("main_end", t.frame)
+        for hart in self.target.harts:
+            self.gdb.select_hart(hart)
+            assertIn("main_end", self.gdb.where())
             # Check register values.
-            self.gdb.thread(t)
             hart_id = self.gdb.p("$x1")
             assertNotIn(hart_id, hart_ids)
             hart_ids.append(hart_id)
@@ -505,12 +511,11 @@ class MulticoreRegTest(GdbTest):
             self.gdb.select_hart(hart)
             self.gdb.p("$x1=0x%x" % (hart.index * 0x800))
             self.gdb.p("$pc=main_post_csrr")
-        self.gdb.c()
-        for t in self.gdb.threads():
-            assertIn("main_end", t.frame)
+        self.gdb.c_all()
         for hart in self.target.harts:
-            # Check register values.
             self.gdb.select_hart(hart)
+            assertIn("main", self.gdb.where())
+            # Check register values.
             for n in range(1, 32):
                 value = self.gdb.p("$x%d" % n)
                 assertEqual(value, hart.index * 0x800 + n - 1)
index d09b5764edf51e7c2b922f6c70bf0ad8c83a42e5..624eb7106d9aa220387c128961dd163dc428b7b5 100644 (file)
@@ -96,6 +96,8 @@ class Target(object):
                 self.openocd_config_path)
         for i, hart in enumerate(self.harts):
             hart.index = i
+            if not hasattr(hart, 'id'):
+                hart.id = i
             if not hart.name:
                 hart.name = "%s-%d" % (self.name, i)
             # Default link script to <name>.lds
diff --git a/debug/targets/RISC-V/spike-1.cfg b/debug/targets/RISC-V/spike-1.cfg
new file mode 100644 (file)
index 0000000..fc20b53
--- /dev/null
@@ -0,0 +1,16 @@
+adapter_khz     10000
+
+interface remote_bitbang
+remote_bitbang_host $::env(REMOTE_BITBANG_HOST)
+remote_bitbang_port $::env(REMOTE_BITBANG_PORT)
+
+set _CHIPNAME riscv
+jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x10e31913
+
+set _TARGETNAME $_CHIPNAME.cpu
+target create $_TARGETNAME riscv -chain-position $_TARGETNAME
+
+gdb_report_data_abort enable
+
+init
+reset halt
diff --git a/debug/targets/RISC-V/spike-2.cfg b/debug/targets/RISC-V/spike-2.cfg
new file mode 100644 (file)
index 0000000..17526ec
--- /dev/null
@@ -0,0 +1,19 @@
+# Connect to a mult-icore RISC-V target, exposing each hart as a thread.
+adapter_khz     10000
+
+interface remote_bitbang
+remote_bitbang_host $::env(REMOTE_BITBANG_HOST)
+remote_bitbang_port $::env(REMOTE_BITBANG_PORT)
+
+set _CHIPNAME riscv
+jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x10e31913
+
+set _TARGETNAME_0 $_CHIPNAME.cpu0
+set _TARGETNAME_1 $_CHIPNAME.cpu1
+target create $_TARGETNAME_0 riscv -chain-position $_CHIPNAME.cpu -coreid 0
+target create $_TARGETNAME_1 riscv -chain-position $_CHIPNAME.cpu -coreid 1
+
+gdb_report_data_abort enable
+
+init
+reset halt
index 9b1841cb80ce01b9e736f4ae107289a889529f7f..799e3cba3151c493e89f163e4b39bf1752897b7a 100644 (file)
@@ -1,3 +1,4 @@
+# Connect to a mult-icore RISC-V target, exposing each hart as a thread.
 adapter_khz     10000
 
 interface remote_bitbang
diff --git a/debug/targets/RISC-V/spike.cfg b/debug/targets/RISC-V/spike.cfg
deleted file mode 100644 (file)
index fc20b53..0000000
+++ /dev/null
@@ -1,16 +0,0 @@
-adapter_khz     10000
-
-interface remote_bitbang
-remote_bitbang_host $::env(REMOTE_BITBANG_HOST)
-remote_bitbang_port $::env(REMOTE_BITBANG_PORT)
-
-set _CHIPNAME riscv
-jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x10e31913
-
-set _TARGETNAME $_CHIPNAME.cpu
-target create $_TARGETNAME riscv -chain-position $_TARGETNAME
-
-gdb_report_data_abort enable
-
-init
-reset halt
diff --git a/debug/targets/RISC-V/spike32-2-rtos.py b/debug/targets/RISC-V/spike32-2-rtos.py
new file mode 100644 (file)
index 0000000..a7b9a1c
--- /dev/null
@@ -0,0 +1,12 @@
+import targets
+import testlib
+
+import spike32  # pylint: disable=import-error
+
+class spike32_2(targets.Target):
+    harts = [spike32.spike32_hart(), spike32.spike32_hart()]
+    openocd_config_path = "spike-rtos.cfg"
+    timeout_sec = 30
+
+    def create(self):
+        return testlib.Spike(self)
index a7b9a1c0a7a1b244afef8f4980554cbd8ec6b53c..719009dbcf88437eb646528ed24af6d41a4c5eb8 100644 (file)
@@ -5,7 +5,7 @@ import spike32  # pylint: disable=import-error
 
 class spike32_2(targets.Target):
     harts = [spike32.spike32_hart(), spike32.spike32_hart()]
-    openocd_config_path = "spike-rtos.cfg"
+    openocd_config_path = "spike-2.cfg"
     timeout_sec = 30
 
     def create(self):
index bcb58927bc0e55205dffa7997edbfafdf94c05f6..809463cb697c065d365eabd6505d4f042ae07c12 100644 (file)
@@ -11,7 +11,7 @@ class spike32_hart(targets.Hart):
 
 class spike32(targets.Target):
     harts = [spike32_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-1.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/targets/RISC-V/spike64-2-rtos.py b/debug/targets/RISC-V/spike64-2-rtos.py
new file mode 100644 (file)
index 0000000..d65d2ab
--- /dev/null
@@ -0,0 +1,12 @@
+import targets
+import testlib
+
+import spike64  # pylint: disable=import-error
+
+class spike64_2_rtos(targets.Target):
+    harts = [spike64.spike64_hart(), spike64.spike64_hart()]
+    openocd_config_path = "spike-rtos.cfg"
+    timeout_sec = 30
+
+    def create(self):
+        return testlib.Spike(self)
index 4f6f1ff92851d51b686ad0f672ee6557748a0f93..709ebbeabef93cae0ce6cb5a85f5ff946c1a18a1 100644 (file)
@@ -5,7 +5,7 @@ import spike64  # pylint: disable=import-error
 
 class spike64_2(targets.Target):
     harts = [spike64.spike64_hart(), spike64.spike64_hart()]
-    openocd_config_path = "spike-rtos.cfg"
+    openocd_config_path = "spike-2.cfg"
     timeout_sec = 30
 
     def create(self):
index 9c37f877e544f3d081e8d6b0578ed3125728ad45..2cd67a5501c85148a97634ed969d76b6ae9aebe8 100644 (file)
@@ -11,7 +11,7 @@ class spike64_hart(targets.Hart):
 
 class spike64(targets.Target):
     harts = [spike64_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-1.cfg"
     timeout_sec = 30
 
     def create(self):
index 0bdb433a1f36301e7896bb2e36d774fb05d56749..385034b82a9103989059703c7d5a7e39b185b87a 100644 (file)
@@ -1,4 +1,5 @@
 import collections
+import os
 import os.path
 import random
 import re
@@ -51,8 +52,6 @@ def compile(args, xlen=32): # pylint: disable=redefined-builtin
         raise Exception("Compile failed!")
 
 class Spike(object):
-    logname = "spike-%d.log" % os.getpid()
-
     def __init__(self, target, halted=False, timeout=None, with_jtag_gdb=True):
         """Launch spike. Return tuple of its process and the port it's running
         on."""
@@ -68,11 +67,13 @@ class Spike(object):
                 "programs/checksum.c", "programs/tiny-malloc.c",
                 "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE")
         cmd.append(self.infinite_loop)
-        logfile = open(self.logname, "w")
-        logfile.write("+ %s\n" % " ".join(cmd))
-        logfile.flush()
+        self.logfile = tempfile.NamedTemporaryFile(prefix="spike-",
+                suffix=".log")
+        self.logname = self.logfile.name
+        self.logfile.write("+ %s\n" % " ".join(cmd))
+        self.logfile.flush()
         self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
-                stdout=logfile, stderr=logfile)
+                stdout=self.logfile, stderr=self.logfile)
 
         if with_jtag_gdb:
             self.port = None
@@ -223,8 +224,7 @@ class Openocd(object):
         logfile.write("+ %s\n" % " ".join(cmd))
         logfile.flush()
 
-        self.ports = []
-        self.port = None
+        self.gdb_ports = []
         self.process = self.start(cmd, logfile)
 
     def start(self, cmd, logfile):
@@ -238,31 +238,32 @@ class Openocd(object):
             # attempt too early.
             start = time.time()
             messaged = False
+            fd = open(Openocd.logname, "r")
             while True:
-                log = open(Openocd.logname).read()
+                line = fd.readline()
+                if not line:
+                    if not process.poll() is None:
+                        raise Exception("OpenOCD exited early.")
+                    time.sleep(0.1)
+                    continue
+
                 m = re.search(r"Listening on port (\d+) for gdb connections",
-                        log)
+                        line)
                 if m:
-                    if not self.ports:
-                        self.port = int(m.group(1))
-                    self.ports.append(int(m.group(1)))
+                    self.gdb_ports.append(int(m.group(1)))
 
-                if "telnet server disabled" in log:
-                    break
+                if "telnet server disabled" in line:
+                    return process
 
-                if not process.poll() is None:
-                    raise Exception(
-                            "OpenOCD exited before completing riscv_examine()")
                 if not messaged and time.time() - start > 1:
                     messaged = True
                     print "Waiting for OpenOCD to start..."
                 if (time.time() - start) > self.timeout:
-                    raise Exception("ERROR: Timed out waiting for OpenOCD to "
+                    raise Exception("Timed out waiting for OpenOCD to "
                             "listen for gdb")
-            return process
+
         except Exception:
-            header("OpenOCD log")
-            sys.stdout.write(log)
+            print_log(Openocd.logname)
             raise
 
     def __del__(self):
@@ -303,42 +304,109 @@ class CannotAccess(Exception):
         Exception.__init__(self)
         self.address = address
 
-Thread = collections.namedtuple('Thread', ('id', 'target_id', 'name',
-    'frame'))
+Thread = collections.namedtuple('Thread', ('id', 'description', 'target_id',
+    'name', 'frame'))
 
 class Gdb(object):
-    logfile = tempfile.NamedTemporaryFile(prefix="gdb", suffix=".log")
-    logname = logfile.name
-    print "GDB Temporary Log File: %s" % logname
-
-    def __init__(self,
-            cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb")):
-        self.child = pexpect.spawn(cmd)
-        self.child.logfile = open(self.logname, "w")
-        self.child.logfile.write("+ %s\n" % cmd)
-        self.wait()
-        self.command("set confirm off")
-        self.command("set width 0")
-        self.command("set height 0")
-        # Force consistency.
-        self.command("set print entry-values no")
+    """A single gdb class which can interact with one or more gdb instances."""
+
+    # pylint: disable=too-many-public-methods
+
+    def __init__(self, ports,
+            cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb"),
+            binary=None):
+        assert ports
+
+        self.stack = []
+
+        self.logfiles = []
+        self.children = []
+        for port in ports:
+            logfile = tempfile.NamedTemporaryFile(prefix="gdb@%d-" % port,
+                    suffix=".log")
+            self.logfiles.append(logfile)
+            child = pexpect.spawn(cmd)
+            child.logfile = logfile
+            child.logfile.write("+ %s\n" % cmd)
+            self.children.append(child)
+        self.active_child = self.children[0]
+
+        self.harts = {}
+        for port, child in zip(ports, self.children):
+            self.select_child(child)
+            self.wait()
+            self.command("set confirm off")
+            self.command("set width 0")
+            self.command("set height 0")
+            # Force consistency.
+            self.command("set print entry-values no")
+            self.command("target extended-remote localhost:%d" % port)
+            if binary:
+                self.command("file %s" % binary)
+            threads = self.threads()
+            for t in threads:
+                hartid = None
+                if t.name:
+                    m = re.search(r"Hart (\d+)", t.name)
+                    if m:
+                        hartid = int(m.group(1))
+                if hartid is None:
+                    if self.harts:
+                        hartid = max(self.harts) + 1
+                    else:
+                        hartid = 0
+                self.harts[hartid] = (child, t)
+
+    def __del__(self):
+        for child in self.children:
+            del child
+
+    def lognames(self):
+        return [logfile.name for logfile in self.logfiles]
+
+    def select_child(self, child):
+        self.active_child = child
 
     def select_hart(self, hart):
-        output = self.command("thread %d" % (hart.index + 1))
+        child, thread = self.harts[hart.id]
+        self.select_child(child)
+        output = self.command("thread %s" % thread.id)
         assert "Unknown" not in output
 
+    def push_state(self):
+        self.stack.append({
+            'active_child': self.active_child
+            })
+
+    def pop_state(self):
+        state = self.stack.pop()
+        self.active_child = state['active_child']
+
     def wait(self):
         """Wait for prompt."""
-        self.child.expect(r"\(gdb\)")
+        self.active_child.expect(r"\(gdb\)")
 
     def command(self, command, timeout=6000):
         """timeout is in seconds"""
-        self.child.sendline(command)
-        self.child.expect("\n", timeout=timeout)
-        self.child.expect(r"\(gdb\)", timeout=timeout)
-        return self.child.before.strip()
+        self.active_child.sendline(command)
+        self.active_child.expect("\n", timeout=timeout)
+        self.active_child.expect(r"\(gdb\)", timeout=timeout)
+        return self.active_child.before.strip()
+
+    def global_command(self, command):
+        """Execute this command on every gdb that we control."""
+        with PrivateState(self):
+            for child in self.children:
+                self.select_child(child)
+                self.command(command)
 
     def c(self, wait=True, timeout=-1, async=False):
+        """
+        Dumb c command.
+        In RTOS mode, gdb will resume all harts.
+        In multi-gdb mode, this command will just go to the current gdb, so
+        will only resume one hart.
+        """
         if async:
             async = "&"
         else:
@@ -348,13 +416,24 @@ class Gdb(object):
             assert "Continuing" in output
             return output
         else:
-            self.child.sendline("c%s" % async)
-            self.child.expect("Continuing")
+            self.active_child.sendline("c%s" % async)
+            self.active_child.expect("Continuing")
+
+    def c_all(self):
+        """Resume every hart."""
+        with PrivateState(self):
+            for child in self.children:
+                child.sendline("c")
+                child.expect("Continuing")
+
+            # Now wait for them all to halt
+            for child in self.children:
+                child.expect(r"\(gdb\)")
 
     def interrupt(self):
-        self.child.send("\003")
-        self.child.expect(r"\(gdb\)", timeout=6000)
-        return self.child.before.strip()
+        self.active_child.send("\003")
+        self.active_child.expect(r"\(gdb\)", timeout=6000)
+        return self.active_child.before.strip()
 
     def x(self, address, size='w'):
         output = self.command("x/%s %s" % (size, address))
@@ -417,17 +496,32 @@ class Gdb(object):
         threads = []
         for line in output.splitlines():
             m = re.match(
-                    r"[\s\*]*(\d+)\s*Thread (\d+)\s*\(Name: ([^\)]+)\s*(.*)",
-                    line)
+                    r"[\s\*]*(\d+)\s*"
+                    r"(Remote target|Thread (\d+)\s*\(Name: ([^\)]+))"
+                    r"\s*(.*)", line)
             if m:
                 threads.append(Thread(*m.groups()))
-        if not threads:
-            threads.append(Thread('1', '1', 'Default', '???'))
+        assert threads
+        #>>>if not threads:
+        #>>>    threads.append(Thread('1', '1', 'Default', '???'))
         return threads
 
     def thread(self, thread):
         return self.command("thread %s" % thread.id)
 
+    def where(self):
+        return self.command("where 1")
+
+class PrivateState(object):
+    def __init__(self, gdb):
+        self.gdb = gdb
+
+    def __enter__(self):
+        self.gdb.push_state()
+
+    def __exit__(self, _type, _value, _traceback):
+        self.gdb.pop_state()
+
 def run_all_tests(module, target, parsed):
     if not os.path.exists(parsed.logs):
         os.makedirs(parsed.logs)
@@ -469,7 +563,7 @@ def run_tests(parsed, target, todo):
         log_name = os.path.join(parsed.logs, "%s-%s-%s.log" %
                 (time.strftime("%Y%m%d-%H%M%S"), type(target).__name__, name))
         log_fd = open(log_name, 'w')
-        print "Running %s > %s ..." % (name, log_name),
+        print "[%s] Starting > %s" % (name, log_name)
         instance = definition(target, hart)
         sys.stdout.flush()
         log_fd.write("Test: %s\n" % name)
@@ -478,12 +572,12 @@ def run_tests(parsed, target, todo):
         real_stdout = sys.stdout
         sys.stdout = log_fd
         try:
-            result = instance.run()
+            result = instance.run(real_stdout)
             log_fd.write("Result: %s\n" % result)
         finally:
             sys.stdout = real_stdout
             log_fd.write("Time elapsed: %.2fs\n" % (time.time() - start))
-        print "%s in %.2fs" % (result, time.time() - start)
+        print "[%s] %s in %.2fs" % (name, result, time.time() - start)
         if result not in good_results and parsed.print_failures:
             sys.stdout.write(open(log_name).read())
         sys.stdout.flush()
@@ -589,7 +683,7 @@ class BaseTest(object):
     def postMortem(self):
         pass
 
-    def run(self):
+    def run(self, real_stdout):
         """
         If compile_args is set, compile a program and set self.binary.
 
@@ -608,6 +702,8 @@ class BaseTest(object):
 
         try:
             self.classSetup()
+            real_stdout.write("[%s] Temporary logs: %s\n" % (
+                type(self).__name__, ", ".join(self.logs)))
             self.setup()
             result = self.test()    # pylint: disable=no-member
         except TestNotApplicable:
@@ -622,7 +718,12 @@ class BaseTest(object):
                 print e.message
             header("Traceback")
             traceback.print_exc(file=sys.stdout)
-            self.postMortem()
+            try:
+                self.postMortem()
+            except Exception as e:  # pylint: disable=broad-except
+                header("postMortem Exception")
+                print e
+                traceback.print_exc(file=sys.stdout)
             return result
 
         finally:
@@ -645,25 +746,22 @@ class GdbTest(BaseTest):
         BaseTest.classSetup(self)
 
         if gdb_cmd:
-            self.gdb = Gdb(gdb_cmd)
+            self.gdb = Gdb(self.server.gdb_ports, gdb_cmd, binary=self.binary)
         else:
-            self.gdb = Gdb()
+            self.gdb = Gdb(self.server.gdb_ports, binary=self.binary)
 
-        self.logs.append(self.gdb.logname)
+        self.logs += self.gdb.lognames()
 
-        if self.binary:
-            self.gdb.command("file %s" % self.binary)
         if self.target:
-            self.gdb.command("set arch riscv:rv%d" % self.hart.xlen)
-            self.gdb.command("set remotetimeout %d" % self.target.timeout_sec)
-        if self.server.port:
-            self.gdb.command(
-                    "target extended-remote localhost:%d" % self.server.port)
-            self.gdb.select_hart(self.hart)
+            self.gdb.global_command("set arch riscv:rv%d" % self.hart.xlen)
+            self.gdb.global_command("set remotetimeout %d" %
+                    self.target.timeout_sec)
 
         for cmd in self.target.gdb_setup:
             self.gdb.command(cmd)
 
+        self.gdb.select_hart(self.hart)
+
         # FIXME: OpenOCD doesn't handle PRIV now
         #self.gdb.p("$priv=3")