gram/core/multiplexer.py

   1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
   4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
   5 # License: BSD
   6
   7 """LiteDRAM Multiplexer."""
   8
   9 import math
  10 from functools import reduce
  11 from operator import or_, and_
  12
  13 from nmigen import *
  14 from nmigen.asserts import Assert, Assume
  15
  16 from gram.common import *
  17 import gram.stream as stream
  18 from gram.compat import RoundRobin, delayed_enter
  19
  20 __ALL__ = ["Multiplexer"]
  21
  22 class _CommandChooser(Elaboratable):
  23     """Arbitrates between requests, filtering them based on their type
  24
  25     Uses RoundRobin to choose current request, filters requests based on
  26     `want_*` signals.
  27
  28     Parameters
  29     ----------
  30     requests : [Endpoint(cmd_request_rw_layout), ...]
  31         Request streams to consider for arbitration
  32
  33     Attributes
  34     ----------
  35     want_reads : Signal, in
  36         Consider read requests
  37     want_writes : Signal, in
  38         Consider write requests
  39     want_cmds : Signal, in
  40         Consider command requests (without ACT)
  41     want_activates : Signal, in
  42         Also consider ACT commands
  43     cmd : Endpoint(cmd_request_rw_layout)
  44         Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
  45     """
  46
  47     def __init__(self, requests):
  48         self.want_reads = Signal()
  49         self.want_writes = Signal()
  50         self.want_cmds = Signal()
  51         self.want_activates = Signal()
  52
  53         self._requests = requests
  54         a = len(requests[0].a)
  55         ba = len(requests[0].ba)
  56
  57         # cas/ras/we are 0 when valid is inactive
  58         self.cmd = stream.Endpoint(cmd_request_rw_layout(a, ba))
  59         self.ready = Signal(len(requests))
  60
  61     def elaborate(self, platform):
  62         m = Module()
  63
  64         n = len(self._requests)
  65
  66         valids = Signal(n)
  67         for i, request in enumerate(self._requests):
  68             is_act_cmd = request.ras & ~request.cas & ~request.we
  69             command = request.is_cmd & self.want_cmds & (~is_act_cmd | self.want_activates)
  70             read = request.is_read == self.want_reads
  71             write = request.is_write == self.want_writes
  72             m.d.comb += valids[i].eq(request.valid & (command | (read & write)))
  73
  74         arbiter = RoundRobin(n)
  75         m.submodules += arbiter
  76         choices = Array(valids[i] for i in range(n))
  77         m.d.comb += [
  78             arbiter.request.eq(valids),
  79             self.cmd.valid.eq(choices[arbiter.grant])
  80         ]
  81
  82         for name in ["a", "ba", "is_read", "is_write", "is_cmd"]:
  83             choices = Array(getattr(req, name) for req in self._requests)
  84             m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  85
  86         for name in ["cas", "ras", "we"]:
  87             # we should only assert those signals when valid is 1
  88             choices = Array(getattr(req, name) for req in self._requests)
  89             with m.If(self.cmd.valid):
  90                 m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  91
  92         for i, request in enumerate(self._requests):
  93             with m.If(self.cmd.valid & self.cmd.ready & (arbiter.grant == i)):
  94                 m.d.comb += self.ready[i].eq(1)
  95
  96         # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
  97         # command is selected when cmd.ready goes high.
  98         m.d.comb += arbiter.stb.eq(self.cmd.ready | ~self.cmd.valid)
  99
 100         return m
 101
 102     # helpers
 103     def accept(self):
 104         return self.cmd.valid & self.cmd.ready
 105
 106     def activate(self):
 107         return self.cmd.ras & ~self.cmd.cas & ~self.cmd.we
 108
 109     def write(self):
 110         return self.cmd.is_write
 111
 112     def read(self):
 113         return self.cmd.is_read
 114
 115 # _Steerer -----------------------------------------------------------------------------------------
 116
 117
 118 (STEER_NOP, STEER_CMD, STEER_REQ, STEER_REFRESH) = range(4)
 119
 120
 121 class _Steerer(Elaboratable):
 122     """Connects selected request to DFI interface
 123
 124     cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
 125     Rank bits are decoded and used to drive cs_n in multi-rank systems,
 126     STEER_REFRESH always enables all ranks.
 127
 128     Parameters
 129     ----------
 130     commands : [Endpoint(cmd_request_rw_layout), ...]
 131         Command streams to choose from. Must be of len=4 in the order:
 132             NOP, CMD, REQ, REFRESH
 133         NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
 134         always considered invalid (because of lack of the `valid` attribute).
 135     dfi : dfi.Interface
 136         DFI interface connected to PHY
 137
 138     Attributes
 139     ----------
 140     sel : [Signal(range(len(commands))), ...], in
 141         Signals for selecting which request gets connected to the corresponding
 142         DFI phase. The signals should take one of the values from STEER_* to
 143         select given source.
 144     """
 145
 146     def __init__(self, commands, dfi):
 147         assert len(commands) == 4
 148         self._commands = commands
 149         self._dfi = dfi
 150         self.sel = [Signal(range(len(commands))) for i in range(len(dfi.phases))]
 151
 152     def elaborate(self, platform):
 153         m = Module()
 154
 155         commands = self._commands
 156         dfi = self._dfi
 157
 158         def valid_and(cmd, attr):
 159             if not hasattr(cmd, "valid"):
 160                 return 0
 161             else:
 162                 return cmd.valid & cmd.ready & getattr(cmd, attr)
 163
 164         for i, (phase, sel) in enumerate(zip(dfi.phases, self.sel)):
 165             nranks = len(phase.cs_n)
 166             rankbits = log2_int(nranks)
 167             if hasattr(phase, "reset_n"):
 168                 m.d.comb += phase.reset_n.eq(1)
 169             m.d.comb += phase.clk_en.eq(Repl(Signal(reset=1), nranks))
 170             if hasattr(phase, "odt"):
 171                 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
 172                 m.d.comb += phase.odt.eq(Repl(Signal(reset=1), nranks))
 173             if rankbits:
 174                 rank_decoder = Decoder(nranks)
 175                 m.submodules += rank_decoder
 176                 m.d.comb += rank_decoder.i.eq(
 177                     (Array(cmd.ba[-rankbits:] for cmd in commands)[sel]))
 178                 if i == 0:  # Select all ranks on refresh.
 179                     with m.If(sel == STEER_REFRESH):
 180                         m.d.sync += phase.cs_n.eq(0)
 181                     with m.Else():
 182                         m.d.sync += phase.cs_n.eq(~rank_decoder.o)
 183                 else:
 184                     m.d.sync += phase.cs_n.eq(~rank_decoder.o)
 185                 m.d.sync += phase.bank.eq(Array(cmd.ba[:-rankbits]
 186                                                 for cmd in commands)[sel])
 187             else:
 188                 m.d.sync += [
 189                     phase.cs_n.eq(0),
 190                     phase.bank.eq(Array(cmd.ba[:] for cmd in commands)[sel]),
 191                 ]
 192
 193             m.d.sync += [
 194                 phase.address.eq(Array(cmd.a for cmd in commands)[sel]),
 195                 phase.cas_n.eq(~Array(valid_and(cmd, "cas") for cmd in commands)[sel]),
 196                 phase.ras_n.eq(~Array(valid_and(cmd, "ras") for cmd in commands)[sel]),
 197                 phase.we_n.eq(~Array(valid_and(cmd, "we") for cmd in commands)[sel])
 198             ]
 199
 200             rddata_ens = Array(valid_and(cmd, "is_read") for cmd in commands)
 201             wrdata_ens = Array(valid_and(cmd, "is_write") for cmd in commands)
 202             m.d.sync += [
 203                 phase.rddata_en.eq(rddata_ens[sel]),
 204                 phase.wrdata_en.eq(wrdata_ens[sel])
 205             ]
 206
 207         return m
 208
 209 class _AntiStarvation(Elaboratable):
 210     def __init__(self, timeout):
 211         self.en = Signal()
 212         self.max_time = Signal(reset=1)
 213         self._timeout = timeout
 214
 215     def elaborate(self, platform):
 216         m = Module()
 217
 218         if self._timeout > 0:
 219             time = Signal(range(self._timeout))
 220             with m.If(~self.en):
 221                 m.d.sync += time.eq(self._timeout-1)
 222                 with m.If(time == 1):
 223                     m.d.sync += self.max_time.eq(1)
 224                 with m.Else():
 225                     m.d.sync += self.max_time.eq(0)
 226             with m.Elif(time != 0):
 227                 m.d.sync += [
 228                     time.eq(time-1),
 229                     self.max_time.eq(0),
 230                 ]
 231         else:
 232             m.d.comb += self.max_time.eq(0)
 233
 234         if platform == "formal" and self._timeout > 0:
 235             m.d.comb += Assert(self.max_time.implies(time == 0))
 236
 237         return m
 238
 239 class Multiplexer(Elaboratable):
 240     """Multplexes requets from BankMachines to DFI
 241
 242     This module multiplexes requests from BankMachines (and Refresher) and
 243     connects them to DFI. Refresh commands are coordinated between the Refresher
 244     and BankMachines to ensure there are no conflicts. Enforces required timings
 245     between commands (some timings are enforced by BankMachines).
 246
 247     Parameters
 248     ----------
 249     settings : ControllerSettings
 250         Controller settings (with .phy, .geom and .timing settings)
 251     bank_machines : [BankMachine, ...]
 252         Bank machines that generate command requests to the Multiplexer
 253     refresher : Refresher
 254         Generates REFRESH command requests
 255     dfi : dfi.Interface
 256         DFI connected to the PHY
 257     interface : LiteDRAMInterface
 258         Data interface connected directly to LiteDRAMCrossbar
 259     """
 260
 261     def __init__(self,
 262                  settings,
 263                  bank_machines,
 264                  refresher,
 265                  dfi,
 266                  interface):
 267         assert(settings.phy.nphases == len(dfi.phases))
 268         self._settings = settings
 269         self._bank_machines = bank_machines
 270         self._refresher = refresher
 271         self._dfi = dfi
 272         self._interface = interface
 273
 274     def elaborate(self, platform):
 275         m = Module()
 276
 277         settings = self._settings
 278         bank_machines = self._bank_machines
 279         refresher = self._refresher
 280         dfi = self._dfi
 281         interface = self._interface
 282
 283         ras_allowed = Signal(reset=1)
 284         cas_allowed = Signal(reset=1)
 285
 286         # Command choosing -------------------------------------------------------------------------
 287         requests = [bm.cmd for bm in bank_machines]
 288         m.submodules.choose_cmd = choose_cmd = _CommandChooser(requests)
 289         m.submodules.choose_req = choose_req = _CommandChooser(requests)
 290         for i, request in enumerate(requests):
 291             m.d.comb += request.ready.eq(
 292                 choose_cmd.ready[i] | choose_req.ready[i])
 293         if settings.phy.nphases == 1:
 294             # When only 1 phase, use choose_req for all requests
 295             choose_cmd = choose_req
 296             m.d.comb += choose_req.want_cmds.eq(1)
 297             m.d.comb += choose_req.want_activates.eq(ras_allowed)
 298
 299         # Command steering -------------------------------------------------------------------------
 300         nop = Record(cmd_request_layout(settings.geom.addressbits,
 301                                         log2_int(len(bank_machines))))
 302         # nop must be 1st
 303         commands = [nop, choose_cmd.cmd, choose_req.cmd, refresher.cmd]
 304         steerer = _Steerer(commands, dfi)
 305         m.submodules += steerer
 306
 307         # tRRD timing (Row to Row delay) -----------------------------------------------------------
 308         m.submodules.trrdcon = trrdcon = tXXDController(settings.timing.tRRD)
 309         m.d.comb += trrdcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 310
 311         # tFAW timing (Four Activate Window) -------------------------------------------------------
 312         m.submodules.tfawcon = tfawcon = tFAWController(settings.timing.tFAW)
 313         m.d.comb += tfawcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 314
 315         # RAS control ------------------------------------------------------------------------------
 316         m.d.comb += ras_allowed.eq(trrdcon.ready & tfawcon.ready)
 317
 318         # tCCD timing (Column to Column delay) -----------------------------------------------------
 319         m.submodules.tccdcon = tccdcon = tXXDController(settings.timing.tCCD)
 320         m.d.comb += tccdcon.valid.eq(choose_req.accept() & (choose_req.write() | choose_req.read()))
 321
 322         # CAS control ------------------------------------------------------------------------------
 323         m.d.comb += cas_allowed.eq(tccdcon.ready)
 324
 325         # tWTR timing (Write to Read delay) --------------------------------------------------------
 326         write_latency = math.ceil(settings.phy.cwl / settings.phy.nphases)
 327         m.submodules.twtrcon = twtrcon = tXXDController(
 328             settings.timing.tWTR + write_latency +
 329             # tCCD must be added since tWTR begins after the transfer is complete
 330             settings.timing.tCCD if settings.timing.tCCD is not None else 0)
 331         m.d.comb += twtrcon.valid.eq(choose_req.accept() & choose_req.write())
 332
 333         # Read/write turnaround --------------------------------------------------------------------
 334         read_available = Signal()
 335         write_available = Signal()
 336         reads = [req.valid & req.is_read for req in requests]
 337         writes = [req.valid & req.is_write for req in requests]
 338         m.d.comb += [
 339             read_available.eq(reduce(or_, reads)),
 340             write_available.eq(reduce(or_, writes))
 341         ]
 342
 343         # Anti Starvation --------------------------------------------------------------------------
 344         m.submodules.read_antistarvation = read_antistarvation = _AntiStarvation(settings.read_time)
 345         read_time_en = read_antistarvation.en
 346         max_read_time = read_antistarvation.max_time
 347
 348         m.submodules.write_antistarvation = write_antistarvation = _AntiStarvation(settings.write_time)
 349         write_time_en = write_antistarvation.en
 350         max_write_time = write_antistarvation.max_time
 351
 352         # Refresh ----------------------------------------------------------------------------------
 353         m.d.comb += [bm.refresh_req.eq(refresher.cmd.valid)
 354                      for bm in bank_machines]
 355         go_to_refresh = Signal()
 356         bm_refresh_gnts = [bm.refresh_gnt for bm in bank_machines]
 357         m.d.comb += go_to_refresh.eq(reduce(and_, bm_refresh_gnts))
 358
 359         # Datapath ---------------------------------------------------------------------------------
 360         all_rddata = [p.rddata for p in dfi.phases]
 361         all_wrdata = [p.wrdata for p in dfi.phases]
 362         all_wrdata_mask = [p.wrdata_mask for p in dfi.phases]
 363         m.d.comb += [
 364             interface.rdata.eq(Cat(*all_rddata)),
 365             Cat(*all_wrdata).eq(interface.wdata),
 366             Cat(*all_wrdata_mask).eq(~interface.wdata_we)
 367         ]
 368
 369         def steerer_sel(steerer, r_w_n):
 370             r = []
 371             for i in range(settings.phy.nphases):
 372                 s = steerer.sel[i].eq(STEER_NOP)
 373                 if r_w_n == "read":
 374                     if i == settings.phy.rdphase:
 375                         s = steerer.sel[i].eq(STEER_REQ)
 376                     elif i == settings.phy.rdcmdphase:
 377                         s = steerer.sel[i].eq(STEER_CMD)
 378                 elif r_w_n == "write":
 379                     if i == settings.phy.wrphase:
 380                         s = steerer.sel[i].eq(STEER_REQ)
 381                     elif i == settings.phy.wrcmdphase:
 382                         s = steerer.sel[i].eq(STEER_CMD)
 383                 else:
 384                     raise ValueError
 385                 r.append(s)
 386             return r
 387
 388         # Control FSM ------------------------------------------------------------------------------
 389         with m.FSM():
 390             with m.State("Read"):
 391                 m.d.comb += [
 392                     read_time_en.eq(1),
 393                     choose_req.want_reads.eq(1),
 394                     steerer_sel(steerer, "read"),
 395                 ]
 396
 397                 with m.If(settings.phy.nphases == 1):
 398                     m.d.comb += choose_req.cmd.ready.eq(
 399                         cas_allowed & (~choose_req.activate() | ras_allowed))
 400                 with m.Else():
 401                     m.d.comb += [
 402                         choose_cmd.want_activates.eq(ras_allowed),
 403                         choose_cmd.cmd.ready.eq(
 404                             ~choose_cmd.activate() | ras_allowed),
 405                         choose_req.cmd.ready.eq(cas_allowed),
 406                     ]
 407
 408                 with m.If(write_available):
 409                     # TODO: switch only after several cycles of ~read_available?
 410                     with m.If(~read_available | max_read_time):
 411                         m.next = "RTW"
 412
 413                 with m.If(go_to_refresh):
 414                     m.next = "Refresh"
 415
 416             with m.State("Write"):
 417                 m.d.comb += [
 418                     write_time_en.eq(1),
 419                     choose_req.want_writes.eq(1),
 420                     steerer_sel(steerer, "write"),
 421                 ]
 422
 423                 with m.If(settings.phy.nphases == 1):
 424                     m.d.comb += choose_req.cmd.ready.eq(
 425                         cas_allowed & (~choose_req.activate() | ras_allowed))
 426                 with m.Else():
 427                     m.d.comb += [
 428                         choose_cmd.want_activates.eq(ras_allowed),
 429                         choose_cmd.cmd.ready.eq(
 430                             ~choose_cmd.activate() | ras_allowed),
 431                         choose_req.cmd.ready.eq(cas_allowed),
 432                     ]
 433
 434                 with m.If(read_available):
 435                     with m.If(~write_available | max_write_time):
 436                         m.next = "WTR"
 437
 438                 with m.If(go_to_refresh):
 439                     m.next = "Refresh"
 440
 441             with m.State("Refresh"):
 442                 m.d.comb += [
 443                     steerer.sel[0].eq(STEER_REFRESH),
 444                     refresher.cmd.ready.eq(1),
 445                 ]
 446                 with m.If(refresher.cmd.last):
 447                     m.next = "Read"
 448
 449             with m.State("WTR"):
 450                 with m.If(twtrcon.ready):
 451                     m.next = "Read"
 452
 453             # TODO: reduce this, actual limit is around (cl+1)/nphases
 454             delayed_enter(m, "RTW", "Write", settings.phy.read_latency-1)
 455
 456         return m