gram/core/multiplexer.py

   1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
   4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
   5 # License: BSD
   6
   7 """LiteDRAM Multiplexer."""
   8
   9 import math
  10 from functools import reduce
  11 from operator import or_, and_
  12
  13 from nmigen import *
  14
  15 from gram.common import *
  16 import gram.stream as stream
  17 from gram.compat import RoundRobin, delayed_enter
  18
  19 __ALL__ = ["Multiplexer"]
  20
  21 class _CommandChooser(Elaboratable):
  22     """Arbitrates between requests, filtering them based on their type
  23
  24     Uses RoundRobin to choose current request, filters requests based on
  25     `want_*` signals.
  26
  27     Parameters
  28     ----------
  29     requests : [Endpoint(cmd_request_rw_layout), ...]
  30         Request streams to consider for arbitration
  31
  32     Attributes
  33     ----------
  34     want_reads : Signal, in
  35         Consider read requests
  36     want_writes : Signal, in
  37         Consider write requests
  38     want_cmds : Signal, in
  39         Consider command requests (without ACT)
  40     want_activates : Signal, in
  41         Also consider ACT commands
  42     cmd : Endpoint(cmd_request_rw_layout)
  43         Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
  44     """
  45
  46     def __init__(self, requests):
  47         self.want_reads = Signal()
  48         self.want_writes = Signal()
  49         self.want_cmds = Signal()
  50         self.want_activates = Signal()
  51
  52         self._requests = requests
  53         a = len(requests[0].a)
  54         ba = len(requests[0].ba)
  55
  56         # cas/ras/we are 0 when valid is inactive
  57         self.cmd = stream.Endpoint(cmd_request_rw_layout(a, ba))
  58         self.ready = Signal(len(requests))
  59
  60     def elaborate(self, platform):
  61         m = Module()
  62
  63         n = len(self._requests)
  64
  65         valids = Signal(n)
  66         for i, request in enumerate(self._requests):
  67             is_act_cmd = request.ras & ~request.cas & ~request.we
  68             command = request.is_cmd & self.want_cmds & (
  69                 ~is_act_cmd | self.want_activates)
  70             read = request.is_read == self.want_reads
  71             write = request.is_write == self.want_writes
  72             m.d.comb += valids[i].eq(request.valid &
  73                                      (command | (read & write)))
  74
  75         arbiter = RoundRobin(n)
  76         m.submodules += arbiter
  77         choices = Array(valids[i] for i in range(n))
  78         m.d.comb += [
  79             arbiter.request.eq(valids),
  80             self.cmd.valid.eq(choices[arbiter.grant])
  81         ]
  82
  83         for name in ["a", "ba", "is_read", "is_write", "is_cmd"]:
  84             choices = Array(getattr(req, name) for req in self._requests)
  85             m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  86
  87         for name in ["cas", "ras", "we"]:
  88             # we should only assert those signals when valid is 1
  89             choices = Array(getattr(req, name) for req in self._requests)
  90             with m.If(self.cmd.valid):
  91                 m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  92
  93         for i, request in enumerate(self._requests):
  94             # with m.If(self.cmd.valid & self.cmd.ready & (arbiter.grant == i)):
  95             # m.d.comb += request.ready.eq(1) # TODO: this shouldn't be commented
  96             self.ready[i].eq(self.cmd.valid & self.cmd.ready &
  97                              (arbiter.grant == i))
  98
  99         # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
 100         # command is selected when cmd.ready goes high.
 101         m.d.comb += arbiter.stb.eq(self.cmd.ready | ~self.cmd.valid)
 102
 103         return m
 104
 105     # helpers
 106     def accept(self):
 107         return self.cmd.valid & self.cmd.ready
 108
 109     def activate(self):
 110         return self.cmd.ras & ~self.cmd.cas & ~self.cmd.we
 111
 112     def write(self):
 113         return self.cmd.is_write
 114
 115     def read(self):
 116         return self.cmd.is_read
 117
 118 # _Steerer -----------------------------------------------------------------------------------------
 119
 120
 121 (STEER_NOP, STEER_CMD, STEER_REQ, STEER_REFRESH) = range(4)
 122
 123
 124 class _Steerer(Elaboratable):
 125     """Connects selected request to DFI interface
 126
 127     cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
 128     Rank bits are decoded and used to drive cs_n in multi-rank systems,
 129     STEER_REFRESH always enables all ranks.
 130
 131     Parameters
 132     ----------
 133     commands : [Endpoint(cmd_request_rw_layout), ...]
 134         Command streams to choose from. Must be of len=4 in the order:
 135             NOP, CMD, REQ, REFRESH
 136         NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
 137         always considered invalid (because of lack of the `valid` attribute).
 138     dfi : dfi.Interface
 139         DFI interface connected to PHY
 140
 141     Attributes
 142     ----------
 143     sel : [Signal(range(len(commands))), ...], in
 144         Signals for selecting which request gets connected to the corresponding
 145         DFI phase. The signals should take one of the values from STEER_* to
 146         select given source.
 147     """
 148
 149     def __init__(self, commands, dfi):
 150         self._commands = commands
 151         self._dfi = dfi
 152         ncmd = len(commands)
 153         nph = len(dfi.phases)
 154         self.sel = [Signal(range(ncmd)) for i in range(nph)]
 155
 156     def elaborate(self, platform):
 157         m = Module()
 158
 159         commands = self._commands
 160         dfi = self._dfi
 161
 162         def valid_and(cmd, attr):
 163             if not hasattr(cmd, "valid"):
 164                 return 0
 165             else:
 166                 return cmd.valid & cmd.ready & getattr(cmd, attr)
 167
 168         for i, (phase, sel) in enumerate(zip(dfi.phases, self.sel)):
 169             nranks = len(phase.cs_n)
 170             rankbits = log2_int(nranks)
 171             if hasattr(phase, "reset_n"):
 172                 m.d.comb += phase.reset_n.eq(1)
 173             m.d.comb += phase.cke.eq(Repl(Signal(reset=1), nranks))
 174             if hasattr(phase, "odt"):
 175                 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
 176                 m.d.comb += phase.odt.eq(Repl(Signal(reset=1), nranks))
 177             if rankbits:
 178                 rank_decoder = Decoder(nranks)
 179                 m.submodules += rank_decoder
 180                 m.d.comb += rank_decoder.i.eq(
 181                     (Array(cmd.ba[-rankbits:] for cmd in commands)[sel]))
 182                 if i == 0:  # Select all ranks on refresh.
 183                     with m.If(sel == STEER_REFRESH):
 184                         m.d.sync += phase.cs_n.eq(0)
 185                     with m.Else():
 186                         m.d.sync += phase.cs_n.eq(~rank_decoder.o)
 187                 else:
 188                     m.d.sync += phase.cs_n.eq(~rank_decoder.o)
 189                 m.d.sync += phase.bank.eq(Array(cmd.ba[:-rankbits]
 190                                                 for cmd in commands)[sel])
 191             else:
 192                 m.d.sync += [
 193                     phase.cs_n.eq(0),
 194                     phase.bank.eq(Array(cmd.ba[:] for cmd in commands)[sel]),
 195                 ]
 196
 197             m.d.sync += [
 198                 phase.address.eq(Array(cmd.a for cmd in commands)[sel]),
 199                 phase.cas_n.eq(~Array(valid_and(cmd, "cas")
 200                                       for cmd in commands)[sel]),
 201                 phase.ras_n.eq(~Array(valid_and(cmd, "ras")
 202                                       for cmd in commands)[sel]),
 203                 phase.we_n.eq(~Array(valid_and(cmd, "we")
 204                                      for cmd in commands)[sel])
 205             ]
 206
 207             rddata_ens = Array(valid_and(cmd, "is_read") for cmd in commands)
 208             wrdata_ens = Array(valid_and(cmd, "is_write") for cmd in commands)
 209             m.d.sync += [
 210                 phase.rddata_en.eq(rddata_ens[sel]),
 211                 phase.wrdata_en.eq(wrdata_ens[sel])
 212             ]
 213
 214         return m
 215
 216 # Multiplexer --------------------------------------------------------------------------------------
 217
 218
 219 class Multiplexer(Elaboratable):
 220     """Multplexes requets from BankMachines to DFI
 221
 222     This module multiplexes requests from BankMachines (and Refresher) and
 223     connects them to DFI. Refresh commands are coordinated between the Refresher
 224     and BankMachines to ensure there are no conflicts. Enforces required timings
 225     between commands (some timings are enforced by BankMachines).
 226
 227     Parameters
 228     ----------
 229     settings : ControllerSettings
 230         Controller settings (with .phy, .geom and .timing settings)
 231     bank_machines : [BankMachine, ...]
 232         Bank machines that generate command requests to the Multiplexer
 233     refresher : Refresher
 234         Generates REFRESH command requests
 235     dfi : dfi.Interface
 236         DFI connected to the PHY
 237     interface : LiteDRAMInterface
 238         Data interface connected directly to LiteDRAMCrossbar
 239     """
 240
 241     def __init__(self,
 242                  settings,
 243                  bank_machines,
 244                  refresher,
 245                  dfi,
 246                  interface):
 247         assert(settings.phy.nphases == len(dfi.phases))
 248         self._settings = settings
 249         self._bank_machines = bank_machines
 250         self._refresher = refresher
 251         self._dfi = dfi
 252         self._interface = interface
 253
 254     def elaborate(self, platform):
 255         m = Module()
 256
 257         settings = self._settings
 258         bank_machines = self._bank_machines
 259         refresher = self._refresher
 260         dfi = self._dfi
 261         interface = self._interface
 262
 263         ras_allowed = Signal(reset=1)
 264         cas_allowed = Signal(reset=1)
 265
 266         # Command choosing -------------------------------------------------------------------------
 267         requests = [bm.cmd for bm in bank_machines]
 268         m.submodules.choose_cmd = choose_cmd = _CommandChooser(requests)
 269         m.submodules.choose_req = choose_req = _CommandChooser(requests)
 270         for i, request in enumerate(requests):
 271             m.d.comb += request.ready.eq(
 272                 choose_cmd.ready[i] | choose_req.ready[i])
 273         if settings.phy.nphases == 1:
 274             # When only 1 phase, use choose_req for all requests
 275             choose_cmd = choose_req
 276             m.d.comb += choose_req.want_cmds.eq(1)
 277             m.d.comb += choose_req.want_activates.eq(ras_allowed)
 278
 279         # Command steering -------------------------------------------------------------------------
 280         nop = Record(cmd_request_layout(settings.geom.addressbits,
 281                                         log2_int(len(bank_machines))))
 282         # nop must be 1st
 283         commands = [nop, choose_cmd.cmd, choose_req.cmd, refresher.cmd]
 284         steerer = _Steerer(commands, dfi)
 285         m.submodules += steerer
 286
 287         # tRRD timing (Row to Row delay) -----------------------------------------------------------
 288         m.submodules.trrdcon = trrdcon = tXXDController(settings.timing.tRRD)
 289         m.d.comb += trrdcon.valid.eq(choose_cmd.accept()
 290                                      & choose_cmd.activate())
 291
 292         # tFAW timing (Four Activate Window) -------------------------------------------------------
 293         m.submodules.tfawcon = tfawcon = tFAWController(settings.timing.tFAW)
 294         m.d.comb += tfawcon.valid.eq(choose_cmd.accept()
 295                                      & choose_cmd.activate())
 296
 297         # RAS control ------------------------------------------------------------------------------
 298         m.d.comb += ras_allowed.eq(trrdcon.ready & tfawcon.ready)
 299
 300         # tCCD timing (Column to Column delay) -----------------------------------------------------
 301         m.submodules.tccdcon = tccdcon = tXXDController(settings.timing.tCCD)
 302         m.d.comb += tccdcon.valid.eq(choose_req.accept()
 303                                      & (choose_req.write() | choose_req.read()))
 304
 305         # CAS control ------------------------------------------------------------------------------
 306         m.d.comb += cas_allowed.eq(tccdcon.ready)
 307
 308         # tWTR timing (Write to Read delay) --------------------------------------------------------
 309         write_latency = math.ceil(settings.phy.cwl / settings.phy.nphases)
 310         m.submodules.twtrcon = twtrcon = tXXDController(
 311             settings.timing.tWTR + write_latency +
 312             # tCCD must be added since tWTR begins after the transfer is complete
 313             settings.timing.tCCD if settings.timing.tCCD is not None else 0)
 314         m.d.comb += twtrcon.valid.eq(choose_req.accept() & choose_req.write())
 315
 316         # Read/write turnaround --------------------------------------------------------------------
 317         read_available = Signal()
 318         write_available = Signal()
 319         reads = [req.valid & req.is_read for req in requests]
 320         writes = [req.valid & req.is_write for req in requests]
 321         m.d.comb += [
 322             read_available.eq(reduce(or_, reads)),
 323             write_available.eq(reduce(or_, writes))
 324         ]
 325
 326         # Anti Starvation --------------------------------------------------------------------------
 327
 328         def anti_starvation(timeout):
 329             en = Signal()
 330             max_time = Signal()
 331             if timeout:
 332                 t = timeout - 1
 333                 time = Signal(range(t+1))
 334                 m.d.comb += max_time.eq(time == 0)
 335                 with m.If(~en):
 336                     m.d.sync += time.eq(t)
 337                 with m.Elif(~max_time):
 338                     m.d.sync += time.eq(time - 1)
 339             else:
 340                 m.d.comb += max_time.eq(0)
 341             return en, max_time
 342
 343         read_time_en,   max_read_time = anti_starvation(settings.read_time)
 344         write_time_en, max_write_time = anti_starvation(settings.write_time)
 345
 346         # Refresh ----------------------------------------------------------------------------------
 347         m.d.comb += [bm.refresh_req.eq(refresher.cmd.valid)
 348                      for bm in bank_machines]
 349         go_to_refresh = Signal()
 350         bm_refresh_gnts = [bm.refresh_gnt for bm in bank_machines]
 351         m.d.comb += go_to_refresh.eq(reduce(and_, bm_refresh_gnts))
 352
 353         # Datapath ---------------------------------------------------------------------------------
 354         all_rddata = [p.rddata for p in dfi.phases]
 355         all_wrdata = [p.wrdata for p in dfi.phases]
 356         all_wrdata_mask = [p.wrdata_mask for p in dfi.phases]
 357         m.d.comb += [
 358             interface.rdata.eq(Cat(*all_rddata)),
 359             Cat(*all_wrdata).eq(interface.wdata),
 360             Cat(*all_wrdata_mask).eq(~interface.wdata_we)
 361         ]
 362
 363         def steerer_sel(steerer, r_w_n):
 364             r = []
 365             for i in range(settings.phy.nphases):
 366                 s = steerer.sel[i].eq(STEER_NOP)
 367                 if r_w_n == "read":
 368                     if i == settings.phy.rdphase:
 369                         s = steerer.sel[i].eq(STEER_REQ)
 370                     elif i == settings.phy.rdcmdphase:
 371                         s = steerer.sel[i].eq(STEER_CMD)
 372                 elif r_w_n == "write":
 373                     if i == settings.phy.wrphase:
 374                         s = steerer.sel[i].eq(STEER_REQ)
 375                     elif i == settings.phy.wrcmdphase:
 376                         s = steerer.sel[i].eq(STEER_CMD)
 377                 else:
 378                     raise ValueError
 379                 r.append(s)
 380             return r
 381
 382         # Control FSM ------------------------------------------------------------------------------
 383         with m.FSM():
 384             with m.State("Read"):
 385                 m.d.comb += [
 386                     read_time_en.eq(1),
 387                     choose_req.want_reads.eq(1),
 388                     steerer_sel(steerer, "read"),
 389                 ]
 390
 391                 with m.If(settings.phy.nphases == 1):
 392                     m.d.comb += choose_req.cmd.ready.eq(
 393                         cas_allowed & (~choose_req.activate() | ras_allowed))
 394                 with m.Else():
 395                     m.d.comb += [
 396                         choose_cmd.want_activates.eq(ras_allowed),
 397                         choose_cmd.cmd.ready.eq(
 398                             ~choose_cmd.activate() | ras_allowed),
 399                         choose_req.cmd.ready.eq(cas_allowed),
 400                     ]
 401
 402                 with m.If(write_available):
 403                     # TODO: switch only after several cycles of ~read_available?
 404                     with m.If(~read_available | max_read_time):
 405                         m.next = "RTW"
 406
 407                 with m.If(go_to_refresh):
 408                     m.next = "Refresh"
 409
 410             with m.State("Write"):
 411                 m.d.comb += [
 412                     write_time_en.eq(1),
 413                     choose_req.want_writes.eq(1),
 414                     steerer_sel(steerer, "write"),
 415                 ]
 416
 417                 with m.If(settings.phy.nphases == 1):
 418                     m.d.comb += choose_req.cmd.ready.eq(
 419                         cas_allowed & (~choose_req.activate() | ras_allowed))
 420                 with m.Else():
 421                     m.d.comb += [
 422                         choose_cmd.want_activates.eq(ras_allowed),
 423                         choose_cmd.cmd.ready.eq(
 424                             ~choose_cmd.activate() | ras_allowed),
 425                         choose_req.cmd.ready.eq(cas_allowed),
 426                     ]
 427
 428                 with m.If(read_available):
 429                     with m.If(~write_available | max_write_time):
 430                         m.next = "WTR"
 431
 432                 with m.If(go_to_refresh):
 433                     m.next = "Refresh"
 434
 435             with m.State("Refresh"):
 436                 m.d.comb += [
 437                     steerer.sel[0].eq(STEER_REFRESH),
 438                     refresher.cmd.ready.eq(1),
 439                 ]
 440                 with m.If(refresher.cmd.last):
 441                     m.next = "Read"
 442
 443             with m.State("WTR"):
 444                 with m.If(twtrcon.ready):
 445                     m.next = "Read"
 446
 447             # TODO: reduce this, actual limit is around (cl+1)/nphases
 448             delayed_enter(m, "RTW", "Write", settings.phy.read_latency-1)
 449
 450         return m