2 * Copyright (c) 2012-2020 ARM Limited
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
14 * Copyright (c) 2013 Amin Farmahini-Farahani
15 * All rights reserved.
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 #ifndef __MEM_CTRL_HH__
47 #define __MEM_CTRL_HH__
51 #include <unordered_set>
55 #include "base/callback.hh"
56 #include "base/statistics.hh"
57 #include "enums/MemSched.hh"
58 #include "mem/qos/mem_ctrl.hh"
59 #include "mem/qport.hh"
60 #include "params/MemCtrl.hh"
61 #include "sim/eventq.hh"
67 * A burst helper helps organize and manage a packet that is larger than
68 * the memory burst size. A system packet that is larger than the burst size
69 * is split into multiple packets and all those packets point to
70 * a single burst helper such that we know when the whole packet is served.
76 /** Number of bursts requred for a system packet **/
77 const unsigned int burstCount;
79 /** Number of bursts serviced so far for a system packet **/
80 unsigned int burstsServiced;
82 BurstHelper(unsigned int _burstCount)
83 : burstCount(_burstCount), burstsServiced(0)
88 * A memory packet stores packets along with the timestamp of when
89 * the packet entered the queue, and also the decoded address.
95 /** When did request enter the controller */
98 /** When will request leave the controller */
101 /** This comes from the outside world */
104 /** RequestorID associated with the packet */
105 const RequestorID _requestorId;
109 /** Does this packet access DRAM?*/
112 /** Will be populated by address decoder */
118 * Bank id is calculated considering banks in all the ranks
119 * eg: 2 ranks each with 8 banks, then bankId = 0 --> rank0, bank0 and
120 * bankId = 8 --> rank1, bank0
122 const uint16_t bankId;
125 * The starting address of the packet.
126 * This address could be unaligned to burst size boundaries. The
127 * reason is to keep the address offset so we can accurately check
128 * incoming read packets with packets in the write queue.
133 * The size of this dram packet in bytes
134 * It is always equal or smaller than the burst size
139 * A pointer to the BurstHelper if this MemPacket is a split packet
140 * If not a split packet (common case), this is set to NULL
142 BurstHelper* burstHelper;
145 * QoS value of the encapsulated packet read at queuing time
150 * Set the packet QoS value
151 * (interface compatibility with Packet)
153 inline void qosValue(const uint8_t qv) { _qosValue = qv; }
156 * Get the packet QoS value
157 * (interface compatibility with Packet)
159 inline uint8_t qosValue() const { return _qosValue; }
162 * Get the packet RequestorID
163 * (interface compatibility with Packet)
165 inline RequestorID requestorId() const { return _requestorId; }
168 * Get the packet size
169 * (interface compatibility with Packet)
171 inline unsigned int getSize() const { return size; }
174 * Get the packet address
175 * (interface compatibility with Packet)
177 inline Addr getAddr() const { return addr; }
180 * Return true if its a read packet
181 * (interface compatibility with Packet)
183 inline bool isRead() const { return read; }
186 * Return true if its a write packet
187 * (interface compatibility with Packet)
189 inline bool isWrite() const { return !read; }
192 * Return true if its a DRAM access
194 inline bool isDram() const { return dram; }
196 MemPacket(PacketPtr _pkt, bool is_read, bool is_dram, uint8_t _rank,
197 uint8_t _bank, uint32_t _row, uint16_t bank_id, Addr _addr,
199 : entryTime(curTick()), readyTime(curTick()), pkt(_pkt),
200 _requestorId(pkt->requestorId()),
201 read(is_read), dram(is_dram), rank(_rank), bank(_bank), row(_row),
202 bankId(bank_id), addr(_addr), size(_size), burstHelper(NULL),
203 _qosValue(_pkt->qosValue())
208 // The memory packets are store in a multiple dequeue structure,
209 // based on their QoS priority
210 typedef std::deque<MemPacket*> MemPacketQueue;
214 * The memory controller is a single-channel memory controller capturing
215 * the most important timing constraints associated with a
216 * contemporary controller. For multi-channel memory systems, the controller
217 * is combined with a crossbar model, with the channel address
218 * interleaving taking part in the crossbar.
220 * As a basic design principle, this controller
221 * model is not cycle callable, but instead uses events to: 1) decide
222 * when new decisions can be made, 2) when resources become available,
223 * 3) when things are to be considered done, and 4) when to send
224 * things back. The controller interfaces to media specific interfaces
225 * to enable flexible topoloties.
226 * Through these simple principles, the model delivers
227 * high performance, and lots of flexibility, allowing users to
228 * evaluate the system impact of a wide range of memory technologies.
230 * For more details, please see Hansson et al, "Simulating DRAM
231 * controllers for future system architecture exploration",
232 * Proc. ISPASS, 2014. If you use this model as part of your research
233 * please cite the paper.
236 class MemCtrl : public QoS::MemCtrl
240 // For now, make use of a queued response port to avoid dealing with
241 // flow control for the responses being sent back
242 class MemoryPort : public QueuedResponsePort
245 RespPacketQueue queue;
250 MemoryPort(const std::string& name, MemCtrl& _ctrl);
254 Tick recvAtomic(PacketPtr pkt) override;
255 Tick recvAtomicBackdoor(
256 PacketPtr pkt, MemBackdoorPtr &backdoor) override;
258 void recvFunctional(PacketPtr pkt) override;
260 bool recvTimingReq(PacketPtr) override;
262 AddrRangeList getAddrRanges() const override;
267 * Our incoming port, for a multi-ported controller add a crossbar
273 * Remember if the memory system is in timing mode
278 * Remember if we have to retry a request when available.
284 * Bunch of things requires to setup "events" in gem5
285 * When event "respondEvent" occurs for example, the method
286 * processRespondEvent is called; no parameters are allowed
289 void processNextReqEvent();
290 EventFunctionWrapper nextReqEvent;
292 void processRespondEvent();
293 EventFunctionWrapper respondEvent;
296 * Check if the read queue has room for more entries
298 * @param pkt_count The number of entries needed in the read queue
299 * @return true if read queue is full, false otherwise
301 bool readQueueFull(unsigned int pkt_count) const;
304 * Check if the write queue has room for more entries
306 * @param pkt_count The number of entries needed in the write queue
307 * @return true if write queue is full, false otherwise
309 bool writeQueueFull(unsigned int pkt_count) const;
312 * When a new read comes in, first check if the write q has a
313 * pending request to the same address.\ If not, decode the
314 * address to populate rank/bank/row, create one or mutliple
315 * "mem_pkt", and push them to the back of the read queue.\
316 * If this is the only
317 * read request in the system, schedule an event to start
320 * @param pkt The request packet from the outside world
321 * @param pkt_count The number of memory bursts the pkt
322 * @param is_dram Does this packet access DRAM?
323 * translate to. If pkt size is larger then one full burst,
324 * then pkt_count is greater than one.
326 void addToReadQueue(PacketPtr pkt, unsigned int pkt_count, bool is_dram);
329 * Decode the incoming pkt, create a mem_pkt and push to the
330 * back of the write queue. \If the write q length is more than
331 * the threshold specified by the user, ie the queue is beginning
332 * to get full, stop reads, and start draining writes.
334 * @param pkt The request packet from the outside world
335 * @param pkt_count The number of memory bursts the pkt
336 * @param is_dram Does this packet access DRAM?
337 * translate to. If pkt size is larger then one full burst,
338 * then pkt_count is greater than one.
340 void addToWriteQueue(PacketPtr pkt, unsigned int pkt_count, bool is_dram);
343 * Actually do the burst based on media specific access function.
344 * Update bus statistics when complete.
346 * @param mem_pkt The memory packet created from the outside world pkt
348 void doBurstAccess(MemPacket* mem_pkt);
351 * When a packet reaches its "readyTime" in the response Q,
352 * use the "access()" method in AbstractMemory to actually
353 * create the response packet, and send it back to the outside
356 * @param pkt The packet from the outside world
357 * @param static_latency Static latency to add before sending the packet
359 void accessAndRespond(PacketPtr pkt, Tick static_latency);
362 * Determine if there is a packet that can issue.
364 * @param pkt The packet to evaluate
366 bool packetReady(MemPacket* pkt);
369 * Calculate the minimum delay used when scheduling a read-to-write
371 * @param return minimum delay
373 Tick minReadToWriteDataGap();
376 * Calculate the minimum delay used when scheduling a write-to-read
378 * @param return minimum delay
380 Tick minWriteToReadDataGap();
383 * The memory schduler/arbiter - picks which request needs to
384 * go next, based on the specified policy such as FCFS or FR-FCFS
385 * and moves it to the head of the queue.
386 * Prioritizes accesses to the same rank as previous burst unless
387 * controller is switching command type.
389 * @param queue Queued requests to consider
390 * @param extra_col_delay Any extra delay due to a read/write switch
391 * @return an iterator to the selected packet, else queue.end()
393 MemPacketQueue::iterator chooseNext(MemPacketQueue& queue,
394 Tick extra_col_delay);
397 * For FR-FCFS policy reorder the read/write queue depending on row buffer
398 * hits and earliest bursts available in memory
400 * @param queue Queued requests to consider
401 * @param extra_col_delay Any extra delay due to a read/write switch
402 * @return an iterator to the selected packet, else queue.end()
404 MemPacketQueue::iterator chooseNextFRFCFS(MemPacketQueue& queue,
405 Tick extra_col_delay);
408 * Calculate burst window aligned tick
410 * @param cmd_tick Initial tick of command
411 * @return burst window aligned tick
413 Tick getBurstWindow(Tick cmd_tick);
416 * Used for debugging to observe the contents of the queues.
418 void printQs() const;
421 * Burst-align an address.
423 * @param addr The potentially unaligned address
424 * @param is_dram Does this packet access DRAM?
426 * @return An address aligned to a memory burst
428 Addr burstAlign(Addr addr, bool is_dram) const;
431 * The controller's main read and write queues,
432 * with support for QoS reordering
434 std::vector<MemPacketQueue> readQueue;
435 std::vector<MemPacketQueue> writeQueue;
438 * To avoid iterating over the write queue to check for
439 * overlapping transactions, maintain a set of burst addresses
440 * that are currently queued. Since we merge writes to the same
441 * location we never have more than one address to the same burst
444 std::unordered_set<Addr> isInWriteQueue;
447 * Response queue where read packets wait after we're done working
448 * with them, but it's not time to send the response yet. The
449 * responses are stored separately mostly to keep the code clean
450 * and help with events scheduling. For all logical purposes such
451 * as sizing the read queue, this and the main read queue need to
454 std::deque<MemPacket*> respQueue;
457 * Holds count of commands issued in burst window starting at
458 * defined Tick. This is used to ensure that the command bandwidth
459 * does not exceed the allowable media constraints.
461 std::unordered_multiset<Tick> burstTicks;
464 * Create pointer to interface of the actual dram media when connected
466 DRAMInterface* const dram;
469 * Create pointer to interface of the actual nvm media when connected
471 NVMInterface* const nvm;
474 * The following are basic design parameters of the memory
475 * controller, and are initialized based on parameter values.
476 * The rowsPerBank is determined based on the capacity, number of
477 * ranks and banks, the burst size, and the row buffer size.
479 const uint32_t readBufferSize;
480 const uint32_t writeBufferSize;
481 const uint32_t writeHighThreshold;
482 const uint32_t writeLowThreshold;
483 const uint32_t minWritesPerSwitch;
484 uint32_t writesThisTime;
485 uint32_t readsThisTime;
488 * Memory controller configuration initialized based on parameter
491 Enums::MemSched memSchedPolicy;
494 * Pipeline latency of the controller frontend. The frontend
495 * contribution is added to writes (that complete when they are in
496 * the write buffer) and reads that are serviced the write buffer.
498 const Tick frontendLatency;
501 * Pipeline latency of the backend and PHY. Along with the
502 * frontend contribution, this latency is added to reads serviced
505 const Tick backendLatency;
508 * Length of a command window, used to check
511 const Tick commandWindow;
514 * Till when must we wait before issuing next RD/WR burst?
521 * The soonest you have to start thinking about the next request
522 * is the longest access time that can occur before
523 * nextBurstAt. Assuming you need to precharge, open a new row,
524 * and access, it is tRP + tRCD + tCL.
528 struct CtrlStats : public Stats::Group
530 CtrlStats(MemCtrl &ctrl);
532 void regStats() override;
536 // All statistics that the model needs to capture
537 Stats::Scalar readReqs;
538 Stats::Scalar writeReqs;
539 Stats::Scalar readBursts;
540 Stats::Scalar writeBursts;
541 Stats::Scalar servicedByWrQ;
542 Stats::Scalar mergedWrBursts;
543 Stats::Scalar neitherReadNorWriteReqs;
544 // Average queue lengths
545 Stats::Average avgRdQLen;
546 Stats::Average avgWrQLen;
548 Stats::Scalar numRdRetry;
549 Stats::Scalar numWrRetry;
550 Stats::Vector readPktSize;
551 Stats::Vector writePktSize;
552 Stats::Vector rdQLenPdf;
553 Stats::Vector wrQLenPdf;
554 Stats::Histogram rdPerTurnAround;
555 Stats::Histogram wrPerTurnAround;
557 Stats::Scalar bytesReadWrQ;
558 Stats::Scalar bytesReadSys;
559 Stats::Scalar bytesWrittenSys;
561 Stats::Formula avgRdBWSys;
562 Stats::Formula avgWrBWSys;
564 Stats::Scalar totGap;
565 Stats::Formula avgGap;
567 // per-requestor bytes read and written to memory
568 Stats::Vector requestorReadBytes;
569 Stats::Vector requestorWriteBytes;
571 // per-requestor bytes read and written to memory rate
572 Stats::Formula requestorReadRate;
573 Stats::Formula requestorWriteRate;
575 // per-requestor read and write serviced memory accesses
576 Stats::Vector requestorReadAccesses;
577 Stats::Vector requestorWriteAccesses;
579 // per-requestor read and write total memory access latency
580 Stats::Vector requestorReadTotalLat;
581 Stats::Vector requestorWriteTotalLat;
583 // per-requestor raed and write average memory access latency
584 Stats::Formula requestorReadAvgLat;
585 Stats::Formula requestorWriteAvgLat;
591 * Upstream caches need this packet until true is returned, so
592 * hold it for deletion until a subsequent call
594 std::unique_ptr<Packet> pendingDelete;
597 * Select either the read or write queue
599 * @param is_read The current burst is a read, select read queue
600 * @return a reference to the appropriate queue
602 std::vector<MemPacketQueue>& selQueue(bool is_read)
604 return (is_read ? readQueue : writeQueue);
608 * Remove commands that have already issued from burstTicks
610 void pruneBurstTick();
614 MemCtrl(const MemCtrlParams &p);
617 * Ensure that all interfaced have drained commands
619 * @return bool flag, set once drain complete
621 bool allIntfDrained() const;
623 DrainState drain() override;
626 * Check for command bus contention for single cycle command.
627 * If there is contention, shift command to next burst.
628 * Check verifies that the commands issued per burst is less
629 * than a defined max number, maxCommandsPerWindow.
630 * Therefore, contention per cycle is not verified and instead
631 * is done based on a burst window.
633 * @param cmd_tick Initial tick of command, to be verified
634 * @param max_cmds_per_burst Number of commands that can issue
636 * @return tick for command issue without contention
638 Tick verifySingleCmd(Tick cmd_tick, Tick max_cmds_per_burst);
641 * Check for command bus contention for multi-cycle (2 currently)
642 * command. If there is contention, shift command(s) to next burst.
643 * Check verifies that the commands issued per burst is less
644 * than a defined max number, maxCommandsPerWindow.
645 * Therefore, contention per cycle is not verified and instead
646 * is done based on a burst window.
648 * @param cmd_tick Initial tick of command, to be verified
649 * @param max_multi_cmd_split Maximum delay between commands
650 * @param max_cmds_per_burst Number of commands that can issue
652 * @return tick for command issue without contention
654 Tick verifyMultiCmd(Tick cmd_tick, Tick max_cmds_per_burst,
655 Tick max_multi_cmd_split = 0);
658 * Is there a respondEvent scheduled?
660 * @return true if event is scheduled
662 bool respondEventScheduled() const { return respondEvent.scheduled(); }
665 * Is there a read/write burst Event scheduled?
667 * @return true if event is scheduled
669 bool requestEventScheduled() const { return nextReqEvent.scheduled(); }
672 * restart the controller
673 * This can be used by interfaces to restart the
674 * scheduler after maintainence commands complete
676 * @param Tick to schedule next event
678 void restartScheduler(Tick tick) { schedule(nextReqEvent, tick); }
681 * Check the current direction of the memory channel
683 * @param next_state Check either the current or next bus state
684 * @return True when bus is currently in a read state
686 bool inReadBusState(bool next_state) const;
689 * Check the current direction of the memory channel
691 * @param next_state Check either the current or next bus state
692 * @return True when bus is currently in a write state
694 bool inWriteBusState(bool next_state) const;
696 Port &getPort(const std::string &if_name,
697 PortID idx=InvalidPortID) override;
699 virtual void init() override;
700 virtual void startup() override;
701 virtual void drainResume() override;
705 Tick recvAtomic(PacketPtr pkt);
706 Tick recvAtomicBackdoor(PacketPtr pkt, MemBackdoorPtr &backdoor);
707 void recvFunctional(PacketPtr pkt);
708 bool recvTimingReq(PacketPtr pkt);
712 #endif //__MEM_CTRL_HH__