1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
7 """LiteDRAM Multiplexer."""
10 from functools
import reduce
11 from operator
import or_
, and_
14 from nmigen
.asserts
import Assert
, Assume
16 from gram
.common
import *
17 import gram
.stream
as stream
18 from gram
.compat
import RoundRobin
, delayed_enter
20 __ALL__
= ["Multiplexer"]
22 class _CommandChooser(Elaboratable
):
23 """Arbitrates between requests, filtering them based on their type
25 Uses RoundRobin to choose current request, filters requests based on
30 requests : [Endpoint(cmd_request_rw_layout), ...]
31 Request streams to consider for arbitration
35 want_reads : Signal, in
36 Consider read requests
37 want_writes : Signal, in
38 Consider write requests
39 want_cmds : Signal, in
40 Consider command requests (without ACT)
41 want_activates : Signal, in
42 Also consider ACT commands
43 cmd : Endpoint(cmd_request_rw_layout)
44 Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
47 def __init__(self
, requests
):
48 self
.want_reads
= Signal()
49 self
.want_writes
= Signal()
50 self
.want_cmds
= Signal()
51 self
.want_activates
= Signal()
53 self
._requests
= requests
54 a
= len(requests
[0].a
)
55 ba
= len(requests
[0].ba
)
57 # cas/ras/we are 0 when valid is inactive
58 self
.cmd
= stream
.Endpoint(cmd_request_rw_layout(a
, ba
))
59 self
.ready
= Signal(len(requests
))
61 def elaborate(self
, platform
):
64 n
= len(self
._requests
)
67 for i
, request
in enumerate(self
._requests
):
68 is_act_cmd
= request
.ras
& ~request
.cas
& ~request
.we
69 command
= request
.is_cmd
& self
.want_cmds
& (~is_act_cmd | self
.want_activates
)
70 read
= request
.is_read
== self
.want_reads
71 write
= request
.is_write
== self
.want_writes
72 m
.d
.comb
+= valids
[i
].eq(request
.valid
& (command |
(read
& write
)))
74 m
.submodules
.arbiter
= arbiter
= RoundRobin(n
)
75 choices
= Array(valids
[i
] for i
in range(n
))
77 arbiter
.request
.eq(valids
),
78 self
.cmd
.valid
.eq(choices
[arbiter
.grant
])
81 for name
in ["a", "ba", "is_read", "is_write", "is_cmd"]:
82 choices
= Array(getattr(req
, name
) for req
in self
._requests
)
83 m
.d
.comb
+= getattr(self
.cmd
, name
).eq(choices
[arbiter
.grant
])
85 for name
in ["cas", "ras", "we"]:
86 # we should only assert those signals when valid is 1
87 choices
= Array(getattr(req
, name
) for req
in self
._requests
)
88 with m
.If(self
.cmd
.valid
):
89 m
.d
.comb
+= getattr(self
.cmd
, name
).eq(choices
[arbiter
.grant
])
91 for i
, request
in enumerate(self
._requests
):
92 with m
.If(self
.cmd
.valid
& self
.cmd
.ready
& (arbiter
.grant
== i
)):
93 m
.d
.comb
+= self
.ready
[i
].eq(1)
95 # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
96 # command is selected when cmd.ready goes high.
97 m
.d
.comb
+= arbiter
.stb
.eq(self
.cmd
.ready | ~self
.cmd
.valid
)
103 return self
.cmd
.valid
& self
.cmd
.ready
106 return self
.cmd
.ras
& ~self
.cmd
.cas
& ~self
.cmd
.we
109 return self
.cmd
.is_write
112 return self
.cmd
.is_read
114 # _Steerer -----------------------------------------------------------------------------------------
117 (STEER_NOP
, STEER_CMD
, STEER_REQ
, STEER_REFRESH
) = range(4)
120 class _Steerer(Elaboratable
):
121 """Connects selected request to DFI interface
123 cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
124 Rank bits are decoded and used to drive cs_n in multi-rank systems,
125 STEER_REFRESH always enables all ranks.
129 commands : [Endpoint(cmd_request_rw_layout), ...]
130 Command streams to choose from. Must be of len=4 in the order:
131 NOP, CMD, REQ, REFRESH
132 NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
133 always considered invalid (because of lack of the `valid` attribute).
135 DFI interface connected to PHY
139 sel : [Signal(range(len(commands))), ...], in
140 Signals for selecting which request gets connected to the corresponding
141 DFI phase. The signals should take one of the values from STEER_* to
145 def __init__(self
, commands
, dfi
):
146 assert len(commands
) == 4
147 self
._commands
= commands
149 self
.sel
= [Signal(range(len(commands
))) for i
in range(len(dfi
.phases
))]
151 def elaborate(self
, platform
):
154 commands
= self
._commands
157 def valid_and(cmd
, attr
):
158 if not hasattr(cmd
, "valid"):
161 return cmd
.valid
& cmd
.ready
& getattr(cmd
, attr
)
163 for i
, (phase
, sel
) in enumerate(zip(dfi
.phases
, self
.sel
)):
164 nranks
= len(phase
.cs
)
165 rankbits
= log2_int(nranks
)
166 if hasattr(phase
, "reset"):
167 m
.d
.comb
+= phase
.reset
.eq(0)
168 m
.d
.comb
+= phase
.clk_en
.eq(Repl(Signal(reset
=1), nranks
))
169 if hasattr(phase
, "odt"):
170 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
171 m
.d
.comb
+= phase
.odt
.eq(Repl(Signal(reset
=1), nranks
))
173 rank_decoder
= Decoder(nranks
)
174 m
.submodules
+= rank_decoder
175 m
.d
.comb
+= rank_decoder
.i
.eq(
176 (Array(cmd
.ba
[-rankbits
:] for cmd
in commands
)[sel
]))
177 if i
== 0: # Select all ranks on refresh.
178 with m
.If(sel
== STEER_REFRESH
):
179 m
.d
.sync
+= phase
.cs
.eq(1)
181 m
.d
.sync
+= phase
.cs
.eq(rank_decoder
.o
)
183 m
.d
.sync
+= phase
.cs
.eq(rank_decoder
.o
)
184 m
.d
.sync
+= phase
.bank
.eq(Array(cmd
.ba
[:-rankbits
]
185 for cmd
in commands
)[sel
])
189 phase
.bank
.eq(Array(cmd
.ba
[:] for cmd
in commands
)[sel
]),
193 phase
.address
.eq(Array(cmd
.a
for cmd
in commands
)[sel
]),
194 phase
.cas
.eq(Array(valid_and(cmd
, "cas") for cmd
in commands
)[sel
]),
195 phase
.ras
.eq(Array(valid_and(cmd
, "ras") for cmd
in commands
)[sel
]),
196 phase
.we
.eq(Array(valid_and(cmd
, "we") for cmd
in commands
)[sel
])
199 rddata_ens
= Array(valid_and(cmd
, "is_read") for cmd
in commands
)
200 wrdata_ens
= Array(valid_and(cmd
, "is_write") for cmd
in commands
)
202 phase
.rddata_en
.eq(rddata_ens
[sel
]),
203 phase
.wrdata_en
.eq(wrdata_ens
[sel
])
208 class _AntiStarvation(Elaboratable
):
209 def __init__(self
, timeout
):
211 self
.max_time
= Signal(reset
=1)
212 self
._timeout
= timeout
214 def elaborate(self
, platform
):
217 # TODO: timeout=1 fails formal checks
218 assert self
._timeout
!= 1
220 if self
._timeout
> 0:
221 time
= Signal(range(self
._timeout
))
224 time
.eq(self
._timeout
-1),
227 with m
.Elif(time
!= 0):
228 m
.d
.sync
+= time
.eq(time
-1)
229 with m
.If(time
== 1):
230 m
.d
.sync
+= self
.max_time
.eq(1)
232 m
.d
.sync
+= self
.max_time
.eq(0)
234 m
.d
.comb
+= self
.max_time
.eq(0)
236 if platform
== "formal" and self
._timeout
> 0:
237 m
.d
.comb
+= Assert(self
.max_time
== (time
== 0))
241 class Multiplexer(Elaboratable
):
242 """Multplexes requets from BankMachines to DFI
244 This module multiplexes requests from BankMachines (and Refresher) and
245 connects them to DFI. Refresh commands are coordinated between the Refresher
246 and BankMachines to ensure there are no conflicts. Enforces required timings
247 between commands (some timings are enforced by BankMachines).
251 settings : ControllerSettings
252 Controller settings (with .phy, .geom and .timing settings)
253 bank_machines : [BankMachine, ...]
254 Bank machines that generate command requests to the Multiplexer
255 refresher : Refresher
256 Generates REFRESH command requests
258 DFI connected to the PHY
259 interface : LiteDRAMInterface
260 Data interface connected directly to LiteDRAMCrossbar
269 assert(settings
.phy
.nphases
== len(dfi
.phases
))
270 self
._settings
= settings
271 self
._bank
_machines
= bank_machines
272 self
._refresher
= refresher
274 self
._interface
= interface
276 def elaborate(self
, platform
):
279 settings
= self
._settings
280 bank_machines
= self
._bank
_machines
281 refresher
= self
._refresher
283 interface
= self
._interface
285 ras_allowed
= Signal(reset
=1)
286 cas_allowed
= Signal(reset
=1)
288 # Command choosing -------------------------------------------------------------------------
289 requests
= [bm
.cmd
for bm
in bank_machines
]
290 m
.submodules
.choose_cmd
= choose_cmd
= _CommandChooser(requests
)
291 m
.submodules
.choose_req
= choose_req
= _CommandChooser(requests
)
292 for i
, request
in enumerate(requests
):
293 m
.d
.comb
+= request
.ready
.eq(
294 choose_cmd
.ready
[i
] | choose_req
.ready
[i
])
295 if settings
.phy
.nphases
== 1:
296 # When only 1 phase, use choose_req for all requests
297 choose_cmd
= choose_req
298 m
.d
.comb
+= choose_req
.want_cmds
.eq(1)
299 m
.d
.comb
+= choose_req
.want_activates
.eq(ras_allowed
)
301 # Command steering -------------------------------------------------------------------------
302 nop
= Record(cmd_request_layout(settings
.geom
.addressbits
,
303 log2_int(len(bank_machines
))))
305 commands
= [nop
, choose_cmd
.cmd
, choose_req
.cmd
, refresher
.cmd
]
306 m
.submodules
.steerer
= steerer
= _Steerer(commands
, dfi
)
308 # tRRD timing (Row to Row delay) -----------------------------------------------------------
309 m
.submodules
.trrdcon
= trrdcon
= tXXDController(settings
.timing
.tRRD
)
310 m
.d
.comb
+= trrdcon
.valid
.eq(choose_cmd
.accept() & choose_cmd
.activate())
312 # tFAW timing (Four Activate Window) -------------------------------------------------------
313 m
.submodules
.tfawcon
= tfawcon
= tFAWController(settings
.timing
.tFAW
)
314 m
.d
.comb
+= tfawcon
.valid
.eq(choose_cmd
.accept() & choose_cmd
.activate())
316 # RAS control ------------------------------------------------------------------------------
317 m
.d
.comb
+= ras_allowed
.eq(trrdcon
.ready
& tfawcon
.ready
)
319 # tCCD timing (Column to Column delay) -----------------------------------------------------
320 m
.submodules
.tccdcon
= tccdcon
= tXXDController(settings
.timing
.tCCD
)
321 m
.d
.comb
+= tccdcon
.valid
.eq(choose_req
.accept() & (choose_req
.write() | choose_req
.read()))
323 # CAS control ------------------------------------------------------------------------------
324 m
.d
.comb
+= cas_allowed
.eq(tccdcon
.ready
)
326 # tWTR timing (Write to Read delay) --------------------------------------------------------
327 write_latency
= math
.ceil(settings
.phy
.cwl
/ settings
.phy
.nphases
)
328 m
.submodules
.twtrcon
= twtrcon
= tXXDController(
329 settings
.timing
.tWTR
+ write_latency
+
330 # tCCD must be added since tWTR begins after the transfer is complete
331 settings
.timing
.tCCD
if settings
.timing
.tCCD
is not None else 0)
332 m
.d
.comb
+= twtrcon
.valid
.eq(choose_req
.accept() & choose_req
.write())
334 # Read/write turnaround --------------------------------------------------------------------
335 read_available
= Signal()
336 write_available
= Signal()
337 reads
= [req
.valid
& req
.is_read
for req
in requests
]
338 writes
= [req
.valid
& req
.is_write
for req
in requests
]
340 read_available
.eq(reduce(or_
, reads
)),
341 write_available
.eq(reduce(or_
, writes
))
344 # Anti Starvation --------------------------------------------------------------------------
345 m
.submodules
.read_antistarvation
= read_antistarvation
= _AntiStarvation(settings
.read_time
)
346 m
.submodules
.write_antistarvation
= write_antistarvation
= _AntiStarvation(settings
.write_time
)
348 # Refresh ----------------------------------------------------------------------------------
349 m
.d
.comb
+= [bm
.refresh_req
.eq(refresher
.cmd
.valid
)
350 for bm
in bank_machines
]
351 go_to_refresh
= Signal()
352 bm_refresh_gnts
= [bm
.refresh_gnt
for bm
in bank_machines
]
353 m
.d
.comb
+= go_to_refresh
.eq(reduce(and_
, bm_refresh_gnts
))
355 # Datapath ---------------------------------------------------------------------------------
356 all_rddata
= [p
.rddata
for p
in dfi
.phases
]
357 all_wrdata
= [p
.wrdata
for p
in dfi
.phases
]
358 all_wrdata_mask
= [p
.wrdata_mask
for p
in dfi
.phases
]
360 interface
.rdata
.eq(Cat(*all_rddata
)),
361 Cat(*all_wrdata
).eq(interface
.wdata
),
362 Cat(*all_wrdata_mask
).eq(~interface
.wdata_we
)
365 def steerer_sel(steerer
, r_w_n
):
367 for i
in range(settings
.phy
.nphases
):
368 s
= steerer
.sel
[i
].eq(STEER_NOP
)
370 if i
== settings
.phy
.rdphase
:
371 s
= steerer
.sel
[i
].eq(STEER_REQ
)
372 elif i
== settings
.phy
.rdcmdphase
:
373 s
= steerer
.sel
[i
].eq(STEER_CMD
)
374 elif r_w_n
== "write":
375 if i
== settings
.phy
.wrphase
:
376 s
= steerer
.sel
[i
].eq(STEER_REQ
)
377 elif i
== settings
.phy
.wrcmdphase
:
378 s
= steerer
.sel
[i
].eq(STEER_CMD
)
384 # Control FSM ------------------------------------------------------------------------------
386 with m
.State("Read"):
388 read_antistarvation
.en
.eq(1),
389 choose_req
.want_reads
.eq(1),
390 steerer_sel(steerer
, "read"),
393 with m
.If(settings
.phy
.nphases
== 1):
394 m
.d
.comb
+= choose_req
.cmd
.ready
.eq(
395 cas_allowed
& (~choose_req
.activate() | ras_allowed
))
398 choose_cmd
.want_activates
.eq(ras_allowed
),
399 choose_cmd
.cmd
.ready
.eq(
400 ~choose_cmd
.activate() | ras_allowed
),
401 choose_req
.cmd
.ready
.eq(cas_allowed
),
404 with m
.If(write_available
):
405 # TODO: switch only after several cycles of ~read_available?
406 with m
.If(~read_available | read_antistarvation
.max_time
):
409 with m
.If(go_to_refresh
):
412 with m
.State("Write"):
414 write_antistarvation
.en
.eq(1),
415 choose_req
.want_writes
.eq(1),
416 steerer_sel(steerer
, "write"),
419 with m
.If(settings
.phy
.nphases
== 1):
420 m
.d
.comb
+= choose_req
.cmd
.ready
.eq(
421 cas_allowed
& (~choose_req
.activate() | ras_allowed
))
424 choose_cmd
.want_activates
.eq(ras_allowed
),
425 choose_cmd
.cmd
.ready
.eq(
426 ~choose_cmd
.activate() | ras_allowed
),
427 choose_req
.cmd
.ready
.eq(cas_allowed
),
430 with m
.If(read_available
):
431 with m
.If(~write_available | write_antistarvation
.max_time
):
434 with m
.If(go_to_refresh
):
437 with m
.State("Refresh"):
439 steerer
.sel
[0].eq(STEER_REFRESH
),
440 refresher
.cmd
.ready
.eq(1),
442 with m
.If(refresher
.cmd
.last
):
446 with m
.If(twtrcon
.ready
):
449 # TODO: reduce this, actual limit is around (cl+1)/nphases
450 delayed_enter(m
, "RTW", "Write", settings
.phy
.read_latency
-1)