gram/core/multiplexer.py

   1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
   4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
   5 # License: BSD
   6
   7 """LiteDRAM Multiplexer."""
   8
   9 import math
  10 from functools import reduce
  11 from operator import or_, and_
  12
  13 from nmigen import *
  14 from nmigen.asserts import Assert, Assume
  15
  16 from gram.common import *
  17 import gram.stream as stream
  18 from gram.compat import RoundRobin, delayed_enter
  19
  20 __ALL__ = ["Multiplexer"]
  21
  22 class _CommandChooser(Elaboratable):
  23     """Arbitrates between requests, filtering them based on their type
  24
  25     Uses RoundRobin to choose current request, filters requests based on
  26     `want_*` signals.
  27
  28     Parameters
  29     ----------
  30     requests : [Endpoint(cmd_request_rw_layout), ...]
  31         Request streams to consider for arbitration
  32
  33     Attributes
  34     ----------
  35     want_reads : Signal, in
  36         Consider read requests
  37     want_writes : Signal, in
  38         Consider write requests
  39     want_cmds : Signal, in
  40         Consider command requests (without ACT)
  41     want_activates : Signal, in
  42         Also consider ACT commands
  43     cmd : Endpoint(cmd_request_rw_layout)
  44         Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
  45     """
  46
  47     def __init__(self, requests):
  48         self.want_reads = Signal()
  49         self.want_writes = Signal()
  50         self.want_cmds = Signal()
  51         self.want_activates = Signal()
  52
  53         self._requests = requests
  54         a = len(requests[0].a)
  55         ba = len(requests[0].ba)
  56
  57         # cas/ras/we are 0 when valid is inactive
  58         self.cmd = stream.Endpoint(cmd_request_rw_layout(a, ba))
  59         self.ready = Signal(len(requests))
  60
  61     def elaborate(self, platform):
  62         m = Module()
  63
  64         n = len(self._requests)
  65
  66         valids = Signal(n)
  67         for i, request in enumerate(self._requests):
  68             is_act_cmd = request.ras & ~request.cas & ~request.we
  69             command = request.is_cmd & self.want_cmds & (~is_act_cmd | self.want_activates)
  70             read = request.is_read == self.want_reads
  71             write = request.is_write == self.want_writes
  72             m.d.comb += valids[i].eq(request.valid & (command | (read & write)))
  73
  74         m.submodules.arbiter = arbiter = RoundRobin(n)
  75         choices = Array(valids[i] for i in range(n))
  76         m.d.comb += [
  77             arbiter.request.eq(valids),
  78             self.cmd.valid.eq(choices[arbiter.grant])
  79         ]
  80
  81         for name in ["a", "ba", "is_read", "is_write", "is_cmd"]:
  82             choices = Array(getattr(req, name) for req in self._requests)
  83             m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  84
  85         for name in ["cas", "ras", "we"]:
  86             # we should only assert those signals when valid is 1
  87             choices = Array(getattr(req, name) for req in self._requests)
  88             with m.If(self.cmd.valid):
  89                 m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  90
  91         for i, request in enumerate(self._requests):
  92             with m.If(self.cmd.valid & self.cmd.ready & (arbiter.grant == i)):
  93                 m.d.comb += self.ready[i].eq(1)
  94
  95         # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
  96         # command is selected when cmd.ready goes high.
  97         m.d.comb += arbiter.stb.eq(self.cmd.ready | ~self.cmd.valid)
  98
  99         return m
 100
 101     # helpers
 102     def accept(self):
 103         return self.cmd.valid & self.cmd.ready
 104
 105     def activate(self):
 106         return self.cmd.ras & ~self.cmd.cas & ~self.cmd.we
 107
 108     def write(self):
 109         return self.cmd.is_write
 110
 111     def read(self):
 112         return self.cmd.is_read
 113
 114 # _Steerer -----------------------------------------------------------------------------------------
 115
 116
 117 (STEER_NOP, STEER_CMD, STEER_REQ, STEER_REFRESH) = range(4)
 118
 119
 120 class _Steerer(Elaboratable):
 121     """Connects selected request to DFI interface
 122
 123     cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
 124     Rank bits are decoded and used to drive cs_n in multi-rank systems,
 125     STEER_REFRESH always enables all ranks.
 126
 127     Parameters
 128     ----------
 129     commands : [Endpoint(cmd_request_rw_layout), ...]
 130         Command streams to choose from. Must be of len=4 in the order:
 131             NOP, CMD, REQ, REFRESH
 132         NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
 133         always considered invalid (because of lack of the `valid` attribute).
 134     dfi : dfi.Interface
 135         DFI interface connected to PHY
 136
 137     Attributes
 138     ----------
 139     sel : [Signal(range(len(commands))), ...], in
 140         Signals for selecting which request gets connected to the corresponding
 141         DFI phase. The signals should take one of the values from STEER_* to
 142         select given source.
 143     """
 144
 145     def __init__(self, commands, dfi):
 146         assert len(commands) == 4
 147         self._commands = commands
 148         self._dfi = dfi
 149         self.sel = [Signal(range(len(commands))) for i in range(len(dfi.phases))]
 150
 151     def elaborate(self, platform):
 152         m = Module()
 153
 154         commands = self._commands
 155         dfi = self._dfi
 156
 157         def valid_and(cmd, attr):
 158             if not hasattr(cmd, "valid"):
 159                 return 0
 160             else:
 161                 return cmd.valid & cmd.ready & getattr(cmd, attr)
 162
 163         for i, (phase, sel) in enumerate(zip(dfi.phases, self.sel)):
 164             nranks = len(phase.cs)
 165             rankbits = log2_int(nranks)
 166             if hasattr(phase, "reset"):
 167                 m.d.comb += phase.reset.eq(0)
 168             m.d.comb += phase.clk_en.eq(Repl(Signal(reset=1), nranks))
 169             if hasattr(phase, "odt"):
 170                 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
 171                 m.d.comb += phase.odt.eq(Repl(Signal(reset=1), nranks))
 172             if rankbits:
 173                 rank_decoder = Decoder(nranks)
 174                 m.submodules += rank_decoder
 175                 m.d.comb += rank_decoder.i.eq(
 176                     (Array(cmd.ba[-rankbits:] for cmd in commands)[sel]))
 177                 if i == 0:  # Select all ranks on refresh.
 178                     with m.If(sel == STEER_REFRESH):
 179                         m.d.sync += phase.cs.eq(1)
 180                     with m.Else():
 181                         m.d.sync += phase.cs.eq(rank_decoder.o)
 182                 else:
 183                     m.d.sync += phase.cs.eq(rank_decoder.o)
 184                 m.d.sync += phase.bank.eq(Array(cmd.ba[:-rankbits]
 185                                                 for cmd in commands)[sel])
 186             else:
 187                 m.d.sync += [
 188                     phase.cs.eq(1),
 189                     phase.bank.eq(Array(cmd.ba[:] for cmd in commands)[sel]),
 190                 ]
 191
 192             m.d.sync += [
 193                 phase.address.eq(Array(cmd.a for cmd in commands)[sel]),
 194                 phase.cas.eq(Array(valid_and(cmd, "cas") for cmd in commands)[sel]),
 195                 phase.ras.eq(Array(valid_and(cmd, "ras") for cmd in commands)[sel]),
 196                 phase.we.eq(Array(valid_and(cmd, "we") for cmd in commands)[sel])
 197             ]
 198
 199             rddata_ens = Array(valid_and(cmd, "is_read") for cmd in commands)
 200             wrdata_ens = Array(valid_and(cmd, "is_write") for cmd in commands)
 201             m.d.sync += [
 202                 phase.rddata_en.eq(rddata_ens[sel]),
 203                 phase.wrdata_en.eq(wrdata_ens[sel])
 204             ]
 205
 206         return m
 207
 208 class _AntiStarvation(Elaboratable):
 209     def __init__(self, timeout):
 210         self.en = Signal()
 211         self.max_time = Signal(reset=1)
 212         self._timeout = timeout
 213
 214     def elaborate(self, platform):
 215         m = Module()
 216
 217         # TODO: timeout=1 fails formal checks
 218         assert self._timeout != 1
 219
 220         if self._timeout > 0:
 221             time = Signal(range(self._timeout))
 222             with m.If(~self.en):
 223                 m.d.sync += [
 224                     time.eq(self._timeout-1),
 225                     self.max_time.eq(0),
 226                 ]
 227             with m.Elif(time != 0):
 228                 m.d.sync += time.eq(time-1)
 229                 with m.If(time == 1):
 230                     m.d.sync += self.max_time.eq(1)
 231                 with m.Else():
 232                     m.d.sync += self.max_time.eq(0)
 233         else:
 234             m.d.comb += self.max_time.eq(0)
 235
 236         if platform == "formal" and self._timeout > 0:
 237             m.d.comb += Assert(self.max_time == (time == 0))
 238
 239         return m
 240
 241 class Multiplexer(Elaboratable):
 242     """Multplexes requets from BankMachines to DFI
 243
 244     This module multiplexes requests from BankMachines (and Refresher) and
 245     connects them to DFI. Refresh commands are coordinated between the Refresher
 246     and BankMachines to ensure there are no conflicts. Enforces required timings
 247     between commands (some timings are enforced by BankMachines).
 248
 249     Parameters
 250     ----------
 251     settings : ControllerSettings
 252         Controller settings (with .phy, .geom and .timing settings)
 253     bank_machines : [BankMachine, ...]
 254         Bank machines that generate command requests to the Multiplexer
 255     refresher : Refresher
 256         Generates REFRESH command requests
 257     dfi : dfi.Interface
 258         DFI connected to the PHY
 259     interface : LiteDRAMInterface
 260         Data interface connected directly to LiteDRAMCrossbar
 261     """
 262
 263     def __init__(self,
 264                  settings,
 265                  bank_machines,
 266                  refresher,
 267                  dfi,
 268                  interface):
 269         assert(settings.phy.nphases == len(dfi.phases))
 270         self._settings = settings
 271         self._bank_machines = bank_machines
 272         self._refresher = refresher
 273         self._dfi = dfi
 274         self._interface = interface
 275
 276     def elaborate(self, platform):
 277         m = Module()
 278
 279         settings = self._settings
 280         bank_machines = self._bank_machines
 281         refresher = self._refresher
 282         dfi = self._dfi
 283         interface = self._interface
 284
 285         ras_allowed = Signal(reset=1)
 286         cas_allowed = Signal(reset=1)
 287
 288         # Command choosing -------------------------------------------------------------------------
 289         requests = [bm.cmd for bm in bank_machines]
 290         m.submodules.choose_cmd = choose_cmd = _CommandChooser(requests)
 291         m.submodules.choose_req = choose_req = _CommandChooser(requests)
 292         for i, request in enumerate(requests):
 293             m.d.comb += request.ready.eq(
 294                 choose_cmd.ready[i] | choose_req.ready[i])
 295         if settings.phy.nphases == 1:
 296             # When only 1 phase, use choose_req for all requests
 297             choose_cmd = choose_req
 298             m.d.comb += choose_req.want_cmds.eq(1)
 299             m.d.comb += choose_req.want_activates.eq(ras_allowed)
 300
 301         # Command steering -------------------------------------------------------------------------
 302         nop = Record(cmd_request_layout(settings.geom.addressbits,
 303                                         log2_int(len(bank_machines))))
 304         # nop must be 1st
 305         commands = [nop, choose_cmd.cmd, choose_req.cmd, refresher.cmd]
 306         m.submodules.steerer = steerer = _Steerer(commands, dfi)
 307
 308         # tRRD timing (Row to Row delay) -----------------------------------------------------------
 309         m.submodules.trrdcon = trrdcon = tXXDController(settings.timing.tRRD)
 310         m.d.comb += trrdcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 311
 312         # tFAW timing (Four Activate Window) -------------------------------------------------------
 313         m.submodules.tfawcon = tfawcon = tFAWController(settings.timing.tFAW)
 314         m.d.comb += tfawcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 315
 316         # RAS control ------------------------------------------------------------------------------
 317         m.d.comb += ras_allowed.eq(trrdcon.ready & tfawcon.ready)
 318
 319         # tCCD timing (Column to Column delay) -----------------------------------------------------
 320         m.submodules.tccdcon = tccdcon = tXXDController(settings.timing.tCCD)
 321         m.d.comb += tccdcon.valid.eq(choose_req.accept() & (choose_req.write() | choose_req.read()))
 322
 323         # CAS control ------------------------------------------------------------------------------
 324         m.d.comb += cas_allowed.eq(tccdcon.ready)
 325
 326         # tWTR timing (Write to Read delay) --------------------------------------------------------
 327         write_latency = math.ceil(settings.phy.cwl / settings.phy.nphases)
 328         m.submodules.twtrcon = twtrcon = tXXDController(
 329             settings.timing.tWTR + write_latency +
 330             # tCCD must be added since tWTR begins after the transfer is complete
 331             settings.timing.tCCD if settings.timing.tCCD is not None else 0)
 332         m.d.comb += twtrcon.valid.eq(choose_req.accept() & choose_req.write())
 333
 334         # Read/write turnaround --------------------------------------------------------------------
 335         read_available = Signal()
 336         write_available = Signal()
 337         reads = [req.valid & req.is_read for req in requests]
 338         writes = [req.valid & req.is_write for req in requests]
 339         m.d.comb += [
 340             read_available.eq(reduce(or_, reads)),
 341             write_available.eq(reduce(or_, writes))
 342         ]
 343
 344         # Anti Starvation --------------------------------------------------------------------------
 345         m.submodules.read_antistarvation = read_antistarvation = _AntiStarvation(settings.read_time)
 346         m.submodules.write_antistarvation = write_antistarvation = _AntiStarvation(settings.write_time)
 347
 348         # Refresh ----------------------------------------------------------------------------------
 349         m.d.comb += [bm.refresh_req.eq(refresher.cmd.valid)
 350                      for bm in bank_machines]
 351         go_to_refresh = Signal()
 352         bm_refresh_gnts = [bm.refresh_gnt for bm in bank_machines]
 353         m.d.comb += go_to_refresh.eq(reduce(and_, bm_refresh_gnts))
 354
 355         # Datapath ---------------------------------------------------------------------------------
 356         all_rddata = [p.rddata for p in dfi.phases]
 357         all_wrdata = [p.wrdata for p in dfi.phases]
 358         all_wrdata_mask = [p.wrdata_mask for p in dfi.phases]
 359         m.d.comb += [
 360             interface.rdata.eq(Cat(*all_rddata)),
 361             Cat(*all_wrdata).eq(interface.wdata),
 362             Cat(*all_wrdata_mask).eq(~interface.wdata_we)
 363         ]
 364
 365         def steerer_sel(steerer, r_w_n):
 366             r = []
 367             for i in range(settings.phy.nphases):
 368                 s = steerer.sel[i].eq(STEER_NOP)
 369                 if r_w_n == "read":
 370                     if i == settings.phy.rdphase:
 371                         s = steerer.sel[i].eq(STEER_REQ)
 372                     elif i == settings.phy.rdcmdphase:
 373                         s = steerer.sel[i].eq(STEER_CMD)
 374                 elif r_w_n == "write":
 375                     if i == settings.phy.wrphase:
 376                         s = steerer.sel[i].eq(STEER_REQ)
 377                     elif i == settings.phy.wrcmdphase:
 378                         s = steerer.sel[i].eq(STEER_CMD)
 379                 else:
 380                     raise ValueError
 381                 r.append(s)
 382             return r
 383
 384         # Control FSM ------------------------------------------------------------------------------
 385         with m.FSM():
 386             with m.State("Read"):
 387                 m.d.comb += [
 388                     read_antistarvation.en.eq(1),
 389                     choose_req.want_reads.eq(1),
 390                     steerer_sel(steerer, "read"),
 391                 ]
 392
 393                 with m.If(settings.phy.nphases == 1):
 394                     m.d.comb += choose_req.cmd.ready.eq(
 395                         cas_allowed & (~choose_req.activate() | ras_allowed))
 396                 with m.Else():
 397                     m.d.comb += [
 398                         choose_cmd.want_activates.eq(ras_allowed),
 399                         choose_cmd.cmd.ready.eq(
 400                             ~choose_cmd.activate() | ras_allowed),
 401                         choose_req.cmd.ready.eq(cas_allowed),
 402                     ]
 403
 404                 with m.If(write_available):
 405                     # TODO: switch only after several cycles of ~read_available?
 406                     with m.If(~read_available | read_antistarvation.max_time):
 407                         m.next = "RTW"
 408
 409                 with m.If(go_to_refresh):
 410                     m.next = "Refresh"
 411
 412             with m.State("Write"):
 413                 m.d.comb += [
 414                     write_antistarvation.en.eq(1),
 415                     choose_req.want_writes.eq(1),
 416                     steerer_sel(steerer, "write"),
 417                 ]
 418
 419                 with m.If(settings.phy.nphases == 1):
 420                     m.d.comb += choose_req.cmd.ready.eq(
 421                         cas_allowed & (~choose_req.activate() | ras_allowed))
 422                 with m.Else():
 423                     m.d.comb += [
 424                         choose_cmd.want_activates.eq(ras_allowed),
 425                         choose_cmd.cmd.ready.eq(
 426                             ~choose_cmd.activate() | ras_allowed),
 427                         choose_req.cmd.ready.eq(cas_allowed),
 428                     ]
 429
 430                 with m.If(read_available):
 431                     with m.If(~write_available | write_antistarvation.max_time):
 432                         m.next = "WTR"
 433
 434                 with m.If(go_to_refresh):
 435                     m.next = "Refresh"
 436
 437             with m.State("Refresh"):
 438                 m.d.comb += [
 439                     steerer.sel[0].eq(STEER_REFRESH),
 440                     refresher.cmd.ready.eq(1),
 441                 ]
 442                 with m.If(refresher.cmd.last):
 443                     m.next = "Read"
 444
 445             with m.State("WTR"):
 446                 with m.If(twtrcon.ready):
 447                     m.next = "Read"
 448
 449             # TODO: reduce this, actual limit is around (cl+1)/nphases
 450             delayed_enter(m, "RTW", "Write", settings.phy.read_latency-1)
 451
 452         return m