gram/core/multiplexer.py

   1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
   4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
   5 # License: BSD
   6
   7 """LiteDRAM Multiplexer."""
   8
   9 import math
  10 from functools import reduce
  11 from operator import or_, and_
  12
  13 from nmigen import *
  14 from nmigen.asserts import Assert, Assume
  15
  16 from gram.common import *
  17 import gram.stream as stream
  18 from gram.compat import RoundRobin, delayed_enter
  19
  20 __ALL__ = ["Multiplexer"]
  21
  22 class _CommandChooser(Elaboratable):
  23     """Arbitrates between requests, filtering them based on their type
  24
  25     Uses RoundRobin to choose current request, filters requests based on
  26     `want_*` signals.
  27
  28     Parameters
  29     ----------
  30     requests : [Endpoint(cmd_request_rw_layout), ...]
  31         Request streams to consider for arbitration
  32
  33     Attributes
  34     ----------
  35     want_reads : Signal, in
  36         Consider read requests
  37     want_writes : Signal, in
  38         Consider write requests
  39     want_cmds : Signal, in
  40         Consider command requests (without ACT)
  41     want_activates : Signal, in
  42         Also consider ACT commands
  43     cmd : Endpoint(cmd_request_rw_layout)
  44         Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
  45     """
  46
  47     def __init__(self, requests):
  48         self.want_reads = Signal()
  49         self.want_writes = Signal()
  50         self.want_cmds = Signal()
  51         self.want_activates = Signal()
  52
  53         self._requests = requests
  54         a = len(requests[0].a)
  55         ba = len(requests[0].ba)
  56
  57         # cas/ras/we are 0 when valid is inactive
  58         self.cmd = stream.Endpoint(cmd_request_rw_layout(a, ba))
  59         self.ready = Signal(len(requests))
  60
  61     def elaborate(self, platform):
  62         m = Module()
  63
  64         n = len(self._requests)
  65
  66         valids = Signal(n)
  67         for i, request in enumerate(self._requests):
  68             is_act_cmd = request.ras & ~request.cas & ~request.we
  69             command = request.is_cmd & self.want_cmds & (~is_act_cmd | self.want_activates)
  70             read = request.is_read == self.want_reads
  71             write = request.is_write == self.want_writes
  72             m.d.comb += valids[i].eq(request.valid & (command | (read & write)))
  73
  74         m.submodules.arbiter = arbiter = RoundRobin(n)
  75         choices = Array(valids[i] for i in range(n))
  76         m.d.comb += [
  77             arbiter.request.eq(valids),
  78             self.cmd.valid.eq(choices[arbiter.grant])
  79         ]
  80
  81         for name in ["a", "ba", "is_read", "is_write", "is_cmd"]:
  82             choices = Array(getattr(req, name) for req in self._requests)
  83             m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  84
  85         for name in ["cas", "ras", "we"]:
  86             # we should only assert those signals when valid is 1
  87             choices = Array(getattr(req, name) for req in self._requests)
  88             with m.If(self.cmd.valid):
  89                 m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  90
  91         for i, request in enumerate(self._requests):
  92             with m.If(self.cmd.valid & self.cmd.ready & (arbiter.grant == i)):
  93                 m.d.comb += self.ready[i].eq(1)
  94
  95         # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
  96         # command is selected when cmd.ready goes high.
  97         m.d.comb += arbiter.stb.eq(self.cmd.ready | ~self.cmd.valid)
  98
  99         return m
 100
 101     # helpers
 102     def accept(self):
 103         return self.cmd.valid & self.cmd.ready
 104
 105     def activate(self):
 106         return self.cmd.ras & ~self.cmd.cas & ~self.cmd.we
 107
 108     def write(self):
 109         return self.cmd.is_write
 110
 111     def read(self):
 112         return self.cmd.is_read
 113
 114 # _Steerer -----------------------------------------------------------------------------------------
 115
 116
 117 (STEER_NOP, STEER_CMD, STEER_REQ, STEER_REFRESH) = range(4)
 118
 119
 120 class _Steerer(Elaboratable):
 121     """Connects selected request to DFI interface
 122
 123     cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
 124     Rank bits are decoded and used to drive cs_n in multi-rank systems,
 125     STEER_REFRESH always enables all ranks.
 126
 127     Parameters
 128     ----------
 129     commands : [Endpoint(cmd_request_rw_layout), ...]
 130         Command streams to choose from. Must be of len=4 in the order:
 131             NOP, CMD, REQ, REFRESH
 132         NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
 133         always considered invalid (because of lack of the `valid` attribute).
 134     dfi : dfi.Interface
 135         DFI interface connected to PHY
 136
 137     Attributes
 138     ----------
 139     sel : [Signal(range(len(commands))), ...], in
 140         Signals for selecting which request gets connected to the corresponding
 141         DFI phase. The signals should take one of the values from STEER_* to
 142         select given source.
 143     """
 144
 145     def __init__(self, commands, dfi):
 146         assert len(commands) == 4
 147         self._commands = commands
 148         self._dfi = dfi
 149         self.sel = [Signal(range(len(commands))) for i in range(len(dfi.phases))]
 150
 151     def elaborate(self, platform):
 152         m = Module()
 153
 154         commands = self._commands
 155         dfi = self._dfi
 156
 157         def valid_and(cmd, attr):
 158             if not hasattr(cmd, "valid"):
 159                 return 0
 160             else:
 161                 return cmd.valid & cmd.ready & getattr(cmd, attr)
 162
 163         for i, (phase, sel) in enumerate(zip(dfi.phases, self.sel)):
 164             nranks = len(phase.cs_n)
 165             rankbits = log2_int(nranks)
 166             if hasattr(phase, "reset_n"):
 167                 m.d.comb += phase.reset_n.eq(1)
 168             m.d.comb += phase.clk_en.eq(Repl(Signal(reset=1), nranks))
 169             if hasattr(phase, "odt"):
 170                 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
 171                 m.d.comb += phase.odt.eq(Repl(Signal(reset=1), nranks))
 172             if rankbits:
 173                 rank_decoder = Decoder(nranks)
 174                 m.submodules += rank_decoder
 175                 m.d.comb += rank_decoder.i.eq(
 176                     (Array(cmd.ba[-rankbits:] for cmd in commands)[sel]))
 177                 if i == 0:  # Select all ranks on refresh.
 178                     with m.If(sel == STEER_REFRESH):
 179                         m.d.sync += phase.cs_n.eq(0)
 180                     with m.Else():
 181                         m.d.sync += phase.cs_n.eq(~rank_decoder.o)
 182                 else:
 183                     m.d.sync += phase.cs_n.eq(~rank_decoder.o)
 184                 m.d.sync += phase.bank.eq(Array(cmd.ba[:-rankbits]
 185                                                 for cmd in commands)[sel])
 186             else:
 187                 m.d.sync += [
 188                     phase.cs_n.eq(0),
 189                     phase.bank.eq(Array(cmd.ba[:] for cmd in commands)[sel]),
 190                 ]
 191
 192             m.d.sync += [
 193                 phase.address.eq(Array(cmd.a for cmd in commands)[sel]),
 194                 phase.cas_n.eq(~Array(valid_and(cmd, "cas") for cmd in commands)[sel]),
 195                 phase.ras_n.eq(~Array(valid_and(cmd, "ras") for cmd in commands)[sel]),
 196                 phase.we_n.eq(~Array(valid_and(cmd, "we") for cmd in commands)[sel])
 197             ]
 198
 199             rddata_ens = Array(valid_and(cmd, "is_read") for cmd in commands)
 200             wrdata_ens = Array(valid_and(cmd, "is_write") for cmd in commands)
 201             m.d.sync += [
 202                 phase.rddata_en.eq(rddata_ens[sel]),
 203                 phase.wrdata_en.eq(wrdata_ens[sel])
 204             ]
 205
 206         return m
 207
 208 class _AntiStarvation(Elaboratable):
 209     def __init__(self, timeout):
 210         self.en = Signal()
 211         self.max_time = Signal(reset=1)
 212         self._timeout = timeout
 213
 214     def elaborate(self, platform):
 215         m = Module()
 216
 217         # TODO: timeout=1 fails formal checks
 218         assert self._timeout != 1
 219
 220         if self._timeout > 0:
 221             time = Signal(range(self._timeout))
 222             with m.If(~self.en):
 223                 m.d.sync += time.eq(self._timeout-1)
 224                 with m.If(time == 1):
 225                     m.d.sync += self.max_time.eq(1)
 226                 with m.Else():
 227                     m.d.sync += self.max_time.eq(0)
 228             with m.Elif(time != 0):
 229                 m.d.sync += [
 230                     time.eq(time-1),
 231                     self.max_time.eq(0),
 232                 ]
 233         else:
 234             m.d.comb += self.max_time.eq(0)
 235
 236         if platform == "formal" and self._timeout > 0:
 237             m.d.comb += Assert(~(self.max_time ^ (time == 0)))
 238
 239         return m
 240
 241 class Multiplexer(Elaboratable):
 242     """Multplexes requets from BankMachines to DFI
 243
 244     This module multiplexes requests from BankMachines (and Refresher) and
 245     connects them to DFI. Refresh commands are coordinated between the Refresher
 246     and BankMachines to ensure there are no conflicts. Enforces required timings
 247     between commands (some timings are enforced by BankMachines).
 248
 249     Parameters
 250     ----------
 251     settings : ControllerSettings
 252         Controller settings (with .phy, .geom and .timing settings)
 253     bank_machines : [BankMachine, ...]
 254         Bank machines that generate command requests to the Multiplexer
 255     refresher : Refresher
 256         Generates REFRESH command requests
 257     dfi : dfi.Interface
 258         DFI connected to the PHY
 259     interface : LiteDRAMInterface
 260         Data interface connected directly to LiteDRAMCrossbar
 261     """
 262
 263     def __init__(self,
 264                  settings,
 265                  bank_machines,
 266                  refresher,
 267                  dfi,
 268                  interface):
 269         assert(settings.phy.nphases == len(dfi.phases))
 270         self._settings = settings
 271         self._bank_machines = bank_machines
 272         self._refresher = refresher
 273         self._dfi = dfi
 274         self._interface = interface
 275
 276     def elaborate(self, platform):
 277         m = Module()
 278
 279         settings = self._settings
 280         bank_machines = self._bank_machines
 281         refresher = self._refresher
 282         dfi = self._dfi
 283         interface = self._interface
 284
 285         ras_allowed = Signal(reset=1)
 286         cas_allowed = Signal(reset=1)
 287
 288         # Command choosing -------------------------------------------------------------------------
 289         requests = [bm.cmd for bm in bank_machines]
 290         m.submodules.choose_cmd = choose_cmd = _CommandChooser(requests)
 291         m.submodules.choose_req = choose_req = _CommandChooser(requests)
 292         for i, request in enumerate(requests):
 293             m.d.comb += request.ready.eq(
 294                 choose_cmd.ready[i] | choose_req.ready[i])
 295         if settings.phy.nphases == 1:
 296             # When only 1 phase, use choose_req for all requests
 297             choose_cmd = choose_req
 298             m.d.comb += choose_req.want_cmds.eq(1)
 299             m.d.comb += choose_req.want_activates.eq(ras_allowed)
 300
 301         # Command steering -------------------------------------------------------------------------
 302         nop = Record(cmd_request_layout(settings.geom.addressbits,
 303                                         log2_int(len(bank_machines))))
 304         # nop must be 1st
 305         commands = [nop, choose_cmd.cmd, choose_req.cmd, refresher.cmd]
 306         m.submodules.steerer = steerer = _Steerer(commands, dfi)
 307
 308         # tRRD timing (Row to Row delay) -----------------------------------------------------------
 309         m.submodules.trrdcon = trrdcon = tXXDController(settings.timing.tRRD)
 310         m.d.comb += trrdcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 311
 312         # tFAW timing (Four Activate Window) -------------------------------------------------------
 313         m.submodules.tfawcon = tfawcon = tFAWController(settings.timing.tFAW)
 314         m.d.comb += tfawcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 315
 316         # RAS control ------------------------------------------------------------------------------
 317         m.d.comb += ras_allowed.eq(trrdcon.ready & tfawcon.ready)
 318
 319         # tCCD timing (Column to Column delay) -----------------------------------------------------
 320         m.submodules.tccdcon = tccdcon = tXXDController(settings.timing.tCCD)
 321         m.d.comb += tccdcon.valid.eq(choose_req.accept() & (choose_req.write() | choose_req.read()))
 322
 323         # CAS control ------------------------------------------------------------------------------
 324         m.d.comb += cas_allowed.eq(tccdcon.ready)
 325
 326         # tWTR timing (Write to Read delay) --------------------------------------------------------
 327         write_latency = math.ceil(settings.phy.cwl / settings.phy.nphases)
 328         m.submodules.twtrcon = twtrcon = tXXDController(
 329             settings.timing.tWTR + write_latency +
 330             # tCCD must be added since tWTR begins after the transfer is complete
 331             settings.timing.tCCD if settings.timing.tCCD is not None else 0)
 332         m.d.comb += twtrcon.valid.eq(choose_req.accept() & choose_req.write())
 333
 334         # Read/write turnaround --------------------------------------------------------------------
 335         read_available = Signal()
 336         write_available = Signal()
 337         reads = [req.valid & req.is_read for req in requests]
 338         writes = [req.valid & req.is_write for req in requests]
 339         m.d.comb += [
 340             read_available.eq(reduce(or_, reads)),
 341             write_available.eq(reduce(or_, writes))
 342         ]
 343
 344         # Anti Starvation --------------------------------------------------------------------------
 345         m.submodules.read_antistarvation = read_antistarvation = _AntiStarvation(settings.read_time)
 346         read_time_en = read_antistarvation.en
 347         max_read_time = read_antistarvation.max_time
 348
 349         m.submodules.write_antistarvation = write_antistarvation = _AntiStarvation(settings.write_time)
 350         write_time_en = write_antistarvation.en
 351         max_write_time = write_antistarvation.max_time
 352
 353         # Refresh ----------------------------------------------------------------------------------
 354         m.d.comb += [bm.refresh_req.eq(refresher.cmd.valid)
 355                      for bm in bank_machines]
 356         go_to_refresh = Signal()
 357         bm_refresh_gnts = [bm.refresh_gnt for bm in bank_machines]
 358         m.d.comb += go_to_refresh.eq(reduce(and_, bm_refresh_gnts))
 359
 360         # Datapath ---------------------------------------------------------------------------------
 361         all_rddata = [p.rddata for p in dfi.phases]
 362         all_wrdata = [p.wrdata for p in dfi.phases]
 363         all_wrdata_mask = [p.wrdata_mask for p in dfi.phases]
 364         m.d.comb += [
 365             interface.rdata.eq(Cat(*all_rddata)),
 366             Cat(*all_wrdata).eq(interface.wdata),
 367             Cat(*all_wrdata_mask).eq(~interface.wdata_we)
 368         ]
 369
 370         def steerer_sel(steerer, r_w_n):
 371             r = []
 372             for i in range(settings.phy.nphases):
 373                 s = steerer.sel[i].eq(STEER_NOP)
 374                 if r_w_n == "read":
 375                     if i == settings.phy.rdphase:
 376                         s = steerer.sel[i].eq(STEER_REQ)
 377                     elif i == settings.phy.rdcmdphase:
 378                         s = steerer.sel[i].eq(STEER_CMD)
 379                 elif r_w_n == "write":
 380                     if i == settings.phy.wrphase:
 381                         s = steerer.sel[i].eq(STEER_REQ)
 382                     elif i == settings.phy.wrcmdphase:
 383                         s = steerer.sel[i].eq(STEER_CMD)
 384                 else:
 385                     raise ValueError
 386                 r.append(s)
 387             return r
 388
 389         # Control FSM ------------------------------------------------------------------------------
 390         with m.FSM():
 391             with m.State("Read"):
 392                 m.d.comb += [
 393                     read_time_en.eq(1),
 394                     choose_req.want_reads.eq(1),
 395                     steerer_sel(steerer, "read"),
 396                 ]
 397
 398                 with m.If(settings.phy.nphases == 1):
 399                     m.d.comb += choose_req.cmd.ready.eq(
 400                         cas_allowed & (~choose_req.activate() | ras_allowed))
 401                 with m.Else():
 402                     m.d.comb += [
 403                         choose_cmd.want_activates.eq(ras_allowed),
 404                         choose_cmd.cmd.ready.eq(
 405                             ~choose_cmd.activate() | ras_allowed),
 406                         choose_req.cmd.ready.eq(cas_allowed),
 407                     ]
 408
 409                 with m.If(write_available):
 410                     # TODO: switch only after several cycles of ~read_available?
 411                     with m.If(~read_available | max_read_time):
 412                         m.next = "RTW"
 413
 414                 with m.If(go_to_refresh):
 415                     m.next = "Refresh"
 416
 417             with m.State("Write"):
 418                 m.d.comb += [
 419                     write_time_en.eq(1),
 420                     choose_req.want_writes.eq(1),
 421                     steerer_sel(steerer, "write"),
 422                 ]
 423
 424                 with m.If(settings.phy.nphases == 1):
 425                     m.d.comb += choose_req.cmd.ready.eq(
 426                         cas_allowed & (~choose_req.activate() | ras_allowed))
 427                 with m.Else():
 428                     m.d.comb += [
 429                         choose_cmd.want_activates.eq(ras_allowed),
 430                         choose_cmd.cmd.ready.eq(
 431                             ~choose_cmd.activate() | ras_allowed),
 432                         choose_req.cmd.ready.eq(cas_allowed),
 433                     ]
 434
 435                 with m.If(read_available):
 436                     with m.If(~write_available | max_write_time):
 437                         m.next = "WTR"
 438
 439                 with m.If(go_to_refresh):
 440                     m.next = "Refresh"
 441
 442             with m.State("Refresh"):
 443                 m.d.comb += [
 444                     steerer.sel[0].eq(STEER_REFRESH),
 445                     refresher.cmd.ready.eq(1),
 446                 ]
 447                 with m.If(refresher.cmd.last):
 448                     m.next = "Read"
 449
 450             with m.State("WTR"):
 451                 with m.If(twtrcon.ready):
 452                     m.next = "Read"
 453
 454             # TODO: reduce this, actual limit is around (cl+1)/nphases
 455             delayed_enter(m, "RTW", "Write", settings.phy.read_latency-1)
 456
 457         return m