From: Andreas Hansson Date: Fri, 9 May 2014 22:58:48 +0000 (-0400) Subject: mem: Ensure DRAM refresh respects timings X-Git-Tag: stable_2014_08_26~24 X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=babf072c1c9d37c5324a65fa1a7ef902c4d8fb43;p=gem5.git mem: Ensure DRAM refresh respects timings This patch adds a state machine for the refresh scheduling to ensure that no accesses are allowed while the refresh is in progress, and that all banks are propely precharged. As part of this change, the precharging of banks of broken out into a method of its own, making is similar to how activations are dealt with. The idle accounting is also updated to ensure that the refresh duration is not added to the time that the DRAM is in the idle state with all banks precharged. --- diff --git a/src/mem/dram_ctrl.cc b/src/mem/dram_ctrl.cc index c701ac616..13ab60e35 100644 --- a/src/mem/dram_ctrl.cc +++ b/src/mem/dram_ctrl.cc @@ -56,8 +56,8 @@ DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : port(name() + ".port", *this), retryRdReq(false), retryWrReq(false), rowHitFlag(false), busState(READ), - respondEvent(this), - refreshEvent(this), nextReqEvent(this), drainManager(NULL), + respondEvent(this), refreshEvent(this), + nextReqEvent(this), drainManager(NULL), deviceBusWidth(p->device_bus_width), burstLength(p->burst_length), deviceRowBufferSize(p->device_rowbuffer_size), devicesPerRank(p->devices_per_rank), @@ -81,8 +81,8 @@ DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : maxAccessesPerRow(p->max_accesses_per_row), frontendLatency(p->static_frontend_latency), backendLatency(p->static_backend_latency), - busBusyUntil(0), prevArrival(0), - nextReqTime(0), startTickPrechargeAll(0), numBanksActive(0) + busBusyUntil(0), refreshDueAt(0), refreshState(REF_IDLE), prevArrival(0), + nextReqTime(0), idleStartTick(0), numBanksActive(0) { // create the bank states based on the dimensions of the ranks and // banks @@ -131,6 +131,12 @@ DRAMCtrl::DRAMCtrl(const DRAMCtrlParams* p) : "address map\n", name()); } } + + // some basic sanity checks + if (tREFI <= tRP || tREFI <= tRFC) { + fatal("tREFI (%d) must be larger than tRP (%d) and tRFC (%d)\n", + tREFI, tRP, tRFC); + } } void @@ -148,7 +154,7 @@ DRAMCtrl::startup() { // update the start tick for the precharge accounting to the // current tick - startTickPrechargeAll = curTick(); + idleStartTick = curTick(); // shift the bus busy time sufficiently far ahead that we never // have to worry about negative values when computing the time for @@ -159,8 +165,9 @@ DRAMCtrl::startup() // print the configuration of the controller printParams(); - // kick off the refresh - schedule(refreshEvent, curTick() + tREFI); + // kick off the refresh, and give ourselves enough time to + // precharge + schedule(refreshEvent, curTick() + tREFI - tRP); } Tick @@ -835,7 +842,7 @@ DRAMCtrl::estimateLatency(DRAMPacket* dram_pkt, Tick inTime) // If the there is no open row (open adaptive), then there // is no precharge delay, otherwise go with tRP - Tick precharge_delay = bank.openRow == -1 ? 0 : tRP; + Tick precharge_delay = bank.openRow == Bank::NO_ROW ? 0 : tRP; //The bank is free, and you may be able to activate potentialActTick = inTime + accLat + precharge_delay; @@ -870,28 +877,39 @@ DRAMCtrl::estimateLatency(DRAMPacket* dram_pkt, Tick inTime) } void -DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank) +DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank, + uint16_t row) { assert(0 <= rank && rank < ranksPerChannel); assert(actTicks[rank].size() == activationLimit); DPRINTF(DRAM, "Activate at tick %d\n", act_tick); - // Tracking accesses after all banks are precharged. - // startTickPrechargeAll: is the tick when all the banks were again - // precharged. The difference between act_tick and startTickPrechargeAll - // gives the time for which DRAM doesn't get any accesses after refreshing - // or after a page is closed in closed-page or open-adaptive-page policy. - if ((numBanksActive == 0) && (act_tick > startTickPrechargeAll)) { - prechargeAllTime += act_tick - startTickPrechargeAll; + // idleStartTick is the tick when all the banks were + // precharged. Thus, the difference between act_tick and + // idleStartTick gives the time for which the DRAM is in an idle + // state with all banks precharged. Note that we may end up + // "changing history" by scheduling an activation before an + // already scheduled precharge, effectively canceling it out. + if (numBanksActive == 0 && act_tick > idleStartTick) { + prechargeAllTime += act_tick - idleStartTick; } - // No need to update number of active banks for closed-page policy as only 1 - // bank will be activated at any given point, which will be instatntly - // precharged - if (pageMgmt == Enums::open || pageMgmt == Enums::open_adaptive || - pageMgmt == Enums::close_adaptive) - ++numBanksActive; + // update the open row + assert(banks[rank][bank].openRow == Bank::NO_ROW); + banks[rank][bank].openRow = row; + + // start counting anew, this covers both the case when we + // auto-precharged, and when this access is forced to + // precharge + banks[rank][bank].bytesAccessed = 0; + banks[rank][bank].rowAccesses = 0; + + ++numBanksActive; + assert(numBanksActive <= banksPerRank * ranksPerChannel); + + DPRINTF(DRAM, "Activate bank at tick %lld, now got %d active\n", + act_tick, numBanksActive); // start by enforcing tRRD for(int i = 0; i < banksPerRank; i++) { @@ -935,6 +953,35 @@ DRAMCtrl::recordActivate(Tick act_tick, uint8_t rank, uint8_t bank) } } +void +DRAMCtrl::prechargeBank(Bank& bank, Tick free_at) +{ + // make sure the bank has an open row + assert(bank.openRow != Bank::NO_ROW); + + // sample the bytes per activate here since we are closing + // the page + bytesPerActivate.sample(bank.bytesAccessed); + + bank.openRow = Bank::NO_ROW; + + bank.freeAt = free_at; + + assert(numBanksActive != 0); + --numBanksActive; + + DPRINTF(DRAM, "Precharged bank, done at tick %lld, now got %d active\n", + bank.freeAt, numBanksActive); + + // if we reached zero, then special conditions apply as we track + // if all banks are precharged for the power models + if (numBanksActive == 0) { + idleStartTick = std::max(idleStartTick, bank.freeAt); + DPRINTF(DRAM, "All banks precharged at tick: %ld\n", + idleStartTick); + } +} + void DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) { @@ -961,30 +1008,30 @@ DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) // Update bank state if (pageMgmt == Enums::open || pageMgmt == Enums::open_adaptive || pageMgmt == Enums::close_adaptive) { - bank.freeAt = curTick() + addDelay + accessLat; - // If you activated a new row do to this access, the next access - // will have to respect tRAS for this bank. - if (!rowHitFlag) { + if (rowHitFlag) { + bank.freeAt = curTick() + addDelay + accessLat; + } else { + // If there is a page open, precharge it. + if (bank.openRow != Bank::NO_ROW) { + prechargeBank(bank, std::max(std::max(bank.freeAt, + bank.tRASDoneAt), + curTick()) + tRP); + } + + // Any precharge is already part of the latency + // estimation, so update the bank free time + bank.freeAt = curTick() + addDelay + accessLat; + // any waiting for banks account for in freeAt actTick = bank.freeAt - tCL - tRCD; + + // If you activated a new row do to this access, the next access + // will have to respect tRAS for this bank bank.tRASDoneAt = actTick + tRAS; - recordActivate(actTick, dram_pkt->rank, dram_pkt->bank); - - // if we closed an open row as a result of this access, - // then sample the number of bytes accessed before - // resetting it - if (bank.openRow != -1) - bytesPerActivate.sample(bank.bytesAccessed); - - // update the open row - bank.openRow = dram_pkt->row; - - // start counting anew, this covers both the case when we - // auto-precharged, and when this access is forced to - // precharge - bank.bytesAccessed = 0; - bank.rowAccesses = 0; + + recordActivate(actTick, dram_pkt->rank, dram_pkt->bank, + dram_pkt->row); } // increment the bytes accessed and the accesses per row @@ -1042,19 +1089,7 @@ DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) // if this access should use auto-precharge, then we are // closing the row if (auto_precharge) { - bank.openRow = -1; - bank.freeAt = std::max(bank.freeAt, bank.tRASDoneAt) + tRP; - --numBanksActive; - if (numBanksActive == 0) { - startTickPrechargeAll = std::max(startTickPrechargeAll, - bank.freeAt); - DPRINTF(DRAM, "All banks precharged at tick: %ld\n", - startTickPrechargeAll); - } - - // sample the bytes per activate here since we are closing - // the page - bytesPerActivate.sample(bank.bytesAccessed); + prechargeBank(bank, std::max(bank.freeAt, bank.tRASDoneAt) + tRP); DPRINTF(DRAM, "Auto-precharged bank: %d\n", dram_pkt->bankId); } @@ -1062,16 +1097,17 @@ DRAMCtrl::doDRAMAccess(DRAMPacket* dram_pkt) DPRINTF(DRAM, "doDRAMAccess::bank.freeAt is %lld\n", bank.freeAt); } else if (pageMgmt == Enums::close) { actTick = curTick() + addDelay + accessLat - tRCD - tCL; - recordActivate(actTick, dram_pkt->rank, dram_pkt->bank); + recordActivate(actTick, dram_pkt->rank, dram_pkt->bank, dram_pkt->row); + + bank.freeAt = actTick + tRCD + tCL; + bank.tRASDoneAt = actTick + tRAS; - // If the DRAM has a very quick tRAS, bank can be made free - // after consecutive tCL,tRCD,tRP times. In general, however, - // an additional wait is required to respect tRAS. - bank.freeAt = std::max(actTick + tRAS + tRP, - actTick + tRCD + tCL + tRP); + // sample the relevant values when precharging + bank.bytesAccessed = burstSize; + bank.rowAccesses = 1; + + prechargeBank(bank, std::max(bank.freeAt, bank.tRASDoneAt) + tRP); DPRINTF(DRAM, "doDRAMAccess::bank.freeAt is %lld\n", bank.freeAt); - bytesPerActivate.sample(burstSize); - startTickPrechargeAll = std::max(startTickPrechargeAll, bank.freeAt); } else panic("No page management policy chosen\n"); @@ -1187,6 +1223,22 @@ DRAMCtrl::processNextReqEvent() busState = READ; } + if (refreshState != REF_IDLE) { + // if a refresh waiting for this event loop to finish, then hand + // over now, and do not schedule a new nextReqEvent + if (refreshState == REF_DRAIN) { + DPRINTF(DRAM, "Refresh drain done, now precharging\n"); + + refreshState = REF_PRE; + + // hand control back to the refresh event loop + schedule(refreshEvent, curTick()); + } + + // let the refresh finish before issuing any further requests + return; + } + // when we get here it is either a read or a write if (busState == READ) { @@ -1298,18 +1350,6 @@ DRAMCtrl::processNextReqEvent() } } -Tick -DRAMCtrl::maxBankFreeAt() const -{ - Tick banksFree = 0; - - for(int i = 0; i < ranksPerChannel; i++) - for(int j = 0; j < banksPerRank; j++) - banksFree = std::max(banks[i][j].freeAt, banksFree); - - return banksFree; -} - uint64_t DRAMCtrl::minBankFreeAt(const deque& queue) const { @@ -1345,21 +1385,97 @@ DRAMCtrl::minBankFreeAt(const deque& queue) const void DRAMCtrl::processRefreshEvent() { - DPRINTF(DRAM, "Refreshing at tick %ld\n", curTick()); + // when first preparing the refresh, remember when it was due + if (refreshState == REF_IDLE) { + // remember when the refresh is due + refreshDueAt = curTick(); + + // proceed to drain + refreshState = REF_DRAIN; + + DPRINTF(DRAM, "Refresh due\n"); + } + + // let any scheduled read or write go ahead, after which it will + // hand control back to this event loop + if (refreshState == REF_DRAIN) { + if (nextReqEvent.scheduled()) { + // hand control over to the request loop until it is + // evaluated next + DPRINTF(DRAM, "Refresh awaiting draining\n"); + + return; + } else { + refreshState = REF_PRE; + } + } - Tick banksFree = std::max(curTick(), maxBankFreeAt()) + tRFC; + // at this point, ensure that all banks are precharged + if (refreshState == REF_PRE) { + DPRINTF(DRAM, "Precharging all\n"); + + // precharge any active bank + for (int i = 0; i < ranksPerChannel; i++) { + for (int j = 0; j < banksPerRank; j++) { + if (banks[i][j].openRow != Bank::NO_ROW) { + // respect both causality and any existing bank + // constraints + Tick free_at = std::max(std::max(banks[i][j].freeAt, + banks[i][j].tRASDoneAt), + curTick()) + tRP; + + prechargeBank(banks[i][j], free_at); + } + } + } + + if (numBanksActive != 0) + panic("Refresh scheduled with %d active banks\n", numBanksActive); + + // advance the state + refreshState = REF_RUN; + + // call ourselves in the future + schedule(refreshEvent, std::max(curTick(), idleStartTick)); + return; + } - for(int i = 0; i < ranksPerChannel; i++) - for(int j = 0; j < banksPerRank; j++) { - banks[i][j].freeAt = banksFree; - banks[i][j].openRow = -1; + // last but not least we perform the actual refresh + if (refreshState == REF_RUN) { + // should never get here with any banks active + assert(numBanksActive == 0); + + Tick banksFree = curTick() + tRFC; + + for (int i = 0; i < ranksPerChannel; i++) { + for (int j = 0; j < banksPerRank; j++) { + banks[i][j].freeAt = banksFree; + } + } + + // make sure we did not wait so long that we cannot make up + // for it + if (refreshDueAt + tREFI < banksFree) { + fatal("Refresh was delayed so long we cannot catch up\n"); } - // updating startTickPrechargeAll, isprechargeAll - numBanksActive = 0; - startTickPrechargeAll = banksFree; + // compensate for the delay in actually performing the refresh + // when scheduling the next one + schedule(refreshEvent, refreshDueAt + tREFI - tRP); + + // back to business as usual + refreshState = REF_IDLE; - schedule(refreshEvent, curTick() + tREFI); + // we are now refreshing until tRFC is done + idleStartTick = banksFree; + + // kick the normal request processing loop into action again + // as early as possible, i.e. when the request is done, the + // scheduling of this event also prevents any new requests + // from going ahead before the scheduled point in time + nextReqTime = banksFree; + schedule(nextReqEvent, nextReqTime); + } } void diff --git a/src/mem/dram_ctrl.hh b/src/mem/dram_ctrl.hh index 8f2e4825e..af802374a 100644 --- a/src/mem/dram_ctrl.hh +++ b/src/mem/dram_ctrl.hh @@ -153,7 +153,7 @@ class DRAMCtrl : public AbstractMemory public: - static const uint32_t INVALID_ROW = -1; + static const uint32_t NO_ROW = -1; uint32_t openRow; @@ -165,7 +165,7 @@ class DRAMCtrl : public AbstractMemory uint32_t bytesAccessed; Bank() : - openRow(INVALID_ROW), freeAt(0), tRASDoneAt(0), actAllowedAt(0), + openRow(NO_ROW), freeAt(0), tRASDoneAt(0), actAllowedAt(0), rowAccesses(0), bytesAccessed(0) { } }; @@ -388,14 +388,6 @@ class DRAMCtrl : public AbstractMemory */ void reorderQueue(std::deque& queue); - /** - * Looking at all banks, determine the moment in time when they - * are all free. - * - * @return The tick when all banks are free - */ - Tick maxBankFreeAt() const; - /** * Find which are the earliest available banks for the enqueued * requests. Assumes maximum of 64 banks per DIMM @@ -411,7 +403,18 @@ class DRAMCtrl : public AbstractMemory * method updates the time that the banks become available based * on the current limits. */ - void recordActivate(Tick act_tick, uint8_t rank, uint8_t bank); + void recordActivate(Tick act_tick, uint8_t rank, uint8_t bank, + uint16_t row); + + /** + * Precharge a given bank and also update when the precharge is + * done. This will also deal with any stats related to the + * accesses to the open page. + * + * @param bank The bank to precharge + * @param free_at Time when the precharge is done + */ + void prechargeBank(Bank& bank, Tick free_at); void printParams() const; @@ -523,6 +526,29 @@ class DRAMCtrl : public AbstractMemory */ Tick busBusyUntil; + /** + * Keep track of when a refresh is due. + */ + Tick refreshDueAt; + + /** + * The refresh state is used to control the progress of the + * refresh scheduling. When normal operation is in progress the + * refresh state is idle. From there, it progresses to the refresh + * drain state once tREFI has passed. The refresh drain state + * captures the DRAM row active state, as it will stay there until + * all ongoing accesses complete. Thereafter all banks are + * precharged, and lastly, the DRAM is refreshed. + */ + enum RefreshState { + REF_IDLE = 0, + REF_DRAIN, + REF_PRE, + REF_RUN + }; + + RefreshState refreshState; + Tick prevArrival; /** @@ -597,8 +623,10 @@ class DRAMCtrl : public AbstractMemory Stats::Formula prechargeAllPercent; Stats::Scalar prechargeAllTime; - // To track number of cycles all the banks are precharged - Tick startTickPrechargeAll; + // To track number of cycles the DRAM is idle, i.e. all the banks + // are precharged + Tick idleStartTick; + // To track number of banks which are currently active unsigned int numBanksActive;