From: Stanislaw Czerniawski Date: Tue, 5 Mar 2019 10:27:56 +0000 (+0000) Subject: dev-arm: Implement a SMMUv3 model X-Git-Tag: v19.0.0.0~780 X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=2574dc41a6b420f0101d0ecf2a3205091ef96940;p=gem5.git dev-arm: Implement a SMMUv3 model This is an implementation of the SMMUv3 architecture. What can it do? - Single-stage and nested translation with 4k or 64k granule. 16k would be straightforward to add. - Large pages are supported. - Works with any gem5 device as long as it is issuing packets with a valid (Sub)StreamId What it can't do? - Fragment stage 1 page when the underlying stage 2 page is smaller. S1 page size > S2 page size is not supported - Invalidations take zero time. This wouldn't be hard to fix. - Checkpointing is not supported - Stall/resume for faulting transactions is not supported Additional contributors: - Michiel W. van Tol - Giacomo Travaglini Change-Id: Ibc606fccd9199b2c1ba739c6335c846ffaa4d564 Signed-off-by: Giacomo Travaglini Reviewed-by: Andreas Sandberg Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/19008 Maintainer: Andreas Sandberg Tested-by: kokoro --- diff --git a/src/dev/arm/SConscript b/src/dev/arm/SConscript index 7d14abe67..0b015e2c3 100644 --- a/src/dev/arm/SConscript +++ b/src/dev/arm/SConscript @@ -44,6 +44,7 @@ if env['TARGET_ISA'] == 'arm': SimObject('FlashDevice.py') SimObject('Gic.py') SimObject('RealView.py') + SimObject('SMMUv3.py') SimObject('UFSHostDevice.py') SimObject('EnergyCtrl.py') SimObject('NoMali.py') @@ -66,6 +67,15 @@ if env['TARGET_ISA'] == 'arm': Source('pl111.cc') Source('hdlcd.cc') Source('kmi.cc') + Source('smmu_v3.cc'); + Source('smmu_v3_caches.cc'); + Source('smmu_v3_cmdexec.cc'); + Source('smmu_v3_events.cc'); + Source('smmu_v3_ports.cc'); + Source('smmu_v3_proc.cc'); + Source('smmu_v3_ptops.cc'); + Source('smmu_v3_slaveifc.cc'); + Source('smmu_v3_transl.cc'); Source('timer_sp804.cc') Source('gpu_nomali.cc') Source('pci_host.cc') @@ -88,6 +98,8 @@ if env['TARGET_ISA'] == 'arm': DebugFlag('GIC') DebugFlag('ITS') DebugFlag('RVCTRL') + DebugFlag('SMMUv3') + DebugFlag('SMMUv3Hazard') DebugFlag('EnergyCtrl') DebugFlag('UFSHostDevice') DebugFlag('VGIC') diff --git a/src/dev/arm/SMMUv3.py b/src/dev/arm/SMMUv3.py new file mode 100644 index 000000000..3c3250832 --- /dev/null +++ b/src/dev/arm/SMMUv3.py @@ -0,0 +1,209 @@ +# Copyright (c) 2013, 2018-2019 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Stan Czerniawski +# Giacomo Travaglini + +from m5.params import * +from m5.proxy import * +from m5.util.fdthelper import * +from m5.SimObject import * +from MemObject import MemObject + +class SMMUv3SlaveInterface(MemObject): + type = 'SMMUv3SlaveInterface' + cxx_header = 'dev/arm/smmu_v3_slaveifc.hh' + + slave = SlavePort('Device port') + ats_master = MasterPort('ATS master port') + ats_slave = SlavePort('ATS slave port') + + port_width = Param.Unsigned(16, 'Port width in bytes (= 1 beat)') + wrbuf_slots = Param.Unsigned(16, 'Write buffer size (in beats)') + xlate_slots = Param.Unsigned(16, 'Translation slots') + + utlb_entries = Param.Unsigned(32, 'Micro TLB size (entries)') + utlb_assoc = Param.Unsigned(0, 'Micro TLB associativity (0=full)') + utlb_policy = Param.String('rr', 'Micro TLB replacement policy') + utlb_enable = Param.Bool(True, 'Micro TLB enable') + utlb_lat = Param.Cycles(1, 'Micro TLB lookup latency') + utlb_slots = Param.Cycles(1, 'Micro TLB lookup slots') + + tlb_entries = Param.Unsigned(2048, 'Main TLB size (entries)') + tlb_assoc = Param.Unsigned(4, 'Main TLB associativity (0=full)') + tlb_policy = Param.String('rr', 'Main TLB replacement policy') + tlb_enable = Param.Bool(True, 'Main TLB enable') + tlb_lat = Param.Cycles(3, 'Main TLB lookup latency') + tlb_slots = Param.Cycles(3, 'Main TLB lookup slots') + + prefetch_enable = Param.Bool(False, + 'Enable prefetch') + prefetch_reserve_last_way = Param.Bool(True, + 'Reserve last way of the main TLB for prefetched entries') + +class SMMUv3(MemObject): + type = 'SMMUv3' + cxx_header = 'dev/arm/smmu_v3.hh' + + master = MasterPort('Master port') + master_walker = MasterPort( + 'Master port for SMMU initiated HWTW requests (optional)') + control = SlavePort('Control port for accessing memory-mapped registers') + sample_period = Param.Clock('10us', 'Stats sample period') + reg_map = Param.AddrRange('Address range for control registers') + system = Param.System(Parent.any, "System this device is part of") + + slave_interfaces = VectorParam.SMMUv3SlaveInterface([], "Slave interfaces") + + # SLAVE INTERFACE<->SMMU link parameters + ifc_smmu_lat = Param.Cycles(8, 'IFC to SMMU communication latency') + smmu_ifc_lat = Param.Cycles(8, 'SMMU to IFC communication latency') + + # SMMU parameters + xlate_slots = Param.Unsigned(64, 'SMMU translation slots') + ptw_slots = Param.Unsigned(16, 'SMMU page table walk slots') + + master_port_width = Param.Unsigned(16, + 'Master port width in bytes (= 1 beat)') + + tlb_entries = Param.Unsigned(2048, 'TLB size (entries)') + tlb_assoc = Param.Unsigned(4, 'TLB associativity (0=full)') + tlb_policy = Param.String('rr', 'TLB replacement policy') + tlb_enable = Param.Bool(False, 'TLB enable') + tlb_lat = Param.Cycles(3, 'TLB lookup latency') + tlb_slots = Param.Cycles(3, 'TLB lookup slots') + + cfg_entries = Param.Unsigned(64, 'Config cache size (entries)') + cfg_assoc = Param.Unsigned(4, 'Config cache associativity (0=full)') + cfg_policy = Param.String('rr', 'Config cache replacement policy') + cfg_enable = Param.Bool(True, 'Config cache enable') + cfg_lat = Param.Cycles(3, 'Config cache lookup latency') + cfg_slots = Param.Cycles(3, 'Config cache lookup slots') + + ipa_entries = Param.Unsigned(128, 'IPA cache size (entries)') + ipa_assoc = Param.Unsigned(4, 'IPA cache associativity (0=full)') + ipa_policy = Param.String('rr', 'IPA cache replacement policy') + ipa_enable = Param.Bool(False, 'IPA cache enable') + ipa_lat = Param.Cycles(3, 'IPA cache lookup lantency') + ipa_slots = Param.Cycles(3, 'IPA cache lookup slots') + + walk_S1L0 = Param.Unsigned(4, 'Walk cache S1L0 size (entries)') + walk_S1L1 = Param.Unsigned(28, 'Walk cache S1L1 size (entries)') + walk_S1L2 = Param.Unsigned(348, 'Walk cache S1L2 size (entries)') + walk_S1L3 = Param.Unsigned(4, 'Walk cache S1L3 size (entries)') + walk_S2L0 = Param.Unsigned(4, 'Walk cache S2L0 size (entries)') + walk_S2L1 = Param.Unsigned(28, 'Walk cache S2L1 size (entries)') + walk_S2L2 = Param.Unsigned(92, 'Walk cache S2L2 size (entries)') + walk_S2L3 = Param.Unsigned(4, 'Walk cache S2L3 size (entries)') + walk_assoc = Param.Unsigned(4, 'Walk cache associativity (0=full)') + walk_policy = Param.String('rr', 'Walk cache replacement policy') + walk_enable = Param.Bool(True, 'Walk cache enable') + wc_nonfinal_enable = Param.Bool(False, + 'Nonfinal translations use walk cache') + wc_s1_levels = Param.Unsigned(7, + 'S1 PT levels cached in walk cache (bit 0 is L0, bit 1 is L1, etc)') + wc_s2_levels = Param.Unsigned(7, + 'S2 PT levels cached in walk cache (bit 0 is L0, bit 1 is L1, etc)') + + walk_lat = Param.Cycles(4, 'Walk cache lookup latency') + walk_slots = Param.Cycles(4, 'Walk cache lookup slots') + + # [28:27] ST_LEVEL = 0b01, 2-level Stream Table supported in addition + # to Linear Stream table. + # [25:24] STALL_MODEL = 0b01, Stall is not supported, all faults + # terminate transaction. + # [22:21] TTENDIAN = 0b10, Endianness support for translation table walks + # (0b10 = Little-endian). + # [19] CD2L = 0b1, 2-level CD table supported. + # [18] VMID16 = 0b1, 16-bit VMID supported. + # [12] ASID16 = 0b1, 16-bit ASID supported. + # [3:2] TTF = 0b10, Translation Table Formats (Stage 1/2) + # (0b10 = AArch64). + # [1] S1P = 0b1, Stage 1 translation supported. + # [0] S2P = 0b1, Stage 2 translation supported. + smmu_idr0 = Param.UInt32(0x094C100F, "SMMU_IDR0 register"); + + # [25:21] CMDQS = 0b00101, Maximum number of Command queue entries + # as log 2 (entries) (0b00101 = 32 entries). + smmu_idr1 = Param.UInt32(0x00A00000, "SMMU_IDR1 register"); + + smmu_idr2 = Param.UInt32(0, "SMMU_IDR2 register"); + smmu_idr3 = Param.UInt32(0, "SMMU_IDR3 register"); + smmu_idr4 = Param.UInt32(0, "SMMU_IDR4 register"); + + # [6] GRAN64K = 0b1, 64KB translation granule supported. + # [4] GRAN4K = 0b1, 4KB translation granule supported. + # [2:0] OAS = 0b101, Output Address Size (0b101 = 48-bit). + smmu_idr5 = Param.UInt32(0x55, "SMMU_IDR5 register"); + smmu_iidr = Param.UInt32(0, "SMMU_IIDR register"); + + # [7:0] (0 = SMMUv3.0) (1 = SMMUv3.1) + smmu_aidr = Param.UInt32(0, "SMMU_AIDR register"); + + def generateDeviceTree(self, state): + reg_addr = self.reg_map.start + reg_size = self.reg_map.size() + node = FdtNode("smmuv3@%x" % long(reg_addr)) + node.appendCompatible("arm,smmu-v3") + node.append(FdtPropertyWords("reg", + state.addrCells(reg_addr) + + state.sizeCells(reg_size))) + node.append(FdtPropertyWords("#iommu-cells", [1])) + + node.appendPhandle(self) + yield node + + def connect(self, device, bus): + """ + Helper method used to connect the SMMU. The master could + be either a dma port (if the SMMU is attached directly to a + dma device), or to a master port (this is the case where the SMMU + is attached to a bridge). + """ + + self.master = bus.slave + self.control = bus.master + + slave_interface = SMMUv3SlaveInterface() + + if hasattr(device, "master"): + slave_interface.slave = device.master + elif hasattr(device, "dma"): + slave_interface.slave = device.dma + else: + print("Unable to attach SMMUv3\n") + sys.exit(1) + + self.slave_interfaces.append(slave_interface) diff --git a/src/dev/arm/amba.hh b/src/dev/arm/amba.hh new file mode 100644 index 000000000..cfc194227 --- /dev/null +++ b/src/dev/arm/amba.hh @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2018 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Giacomo Travaglini + */ + +#ifndef __DEV_ARM_AMBA_HH__ +#define __DEV_ARM_AMBA_HH__ + +#include "mem/packet.hh" + +namespace AMBA +{ + +typedef MasterID OrderID; + +static OrderID +orderId(PacketPtr pkt) +{ + return pkt->req->masterId(); +} + +} // namespace AMBA + +#endif // __DEV_ARM_AMBA_HH__ diff --git a/src/dev/arm/smmu_v3.cc b/src/dev/arm/smmu_v3.cc new file mode 100644 index 000000000..25176491e --- /dev/null +++ b/src/dev/arm/smmu_v3.cc @@ -0,0 +1,781 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#include "dev/arm/smmu_v3.hh" + +#include +#include + +#include "base/bitfield.hh" +#include "base/cast.hh" +#include "base/logging.hh" +#include "base/trace.hh" +#include "base/types.hh" +#include "debug/Checkpoint.hh" +#include "debug/SMMUv3.hh" +#include "dev/arm/smmu_v3_transl.hh" +#include "mem/packet_access.hh" +#include "sim/system.hh" + +SMMUv3::SMMUv3(SMMUv3Params *params) : + MemObject(params), + system(*params->system), + masterId(params->system->getMasterId(this)), + masterPort(name() + ".master", *this), + masterTableWalkPort(name() + ".master_walker", *this), + controlPort(name() + ".control", *this, params->reg_map), + tlb(params->tlb_entries, params->tlb_assoc, params->tlb_policy), + configCache(params->cfg_entries, params->cfg_assoc, params->cfg_policy), + ipaCache(params->ipa_entries, params->ipa_assoc, params->ipa_policy), + walkCache({ { params->walk_S1L0, params->walk_S1L1, + params->walk_S1L2, params->walk_S1L3, + params->walk_S2L0, params->walk_S2L1, + params->walk_S2L2, params->walk_S2L3 } }, + params->walk_assoc, params->walk_policy), + tlbEnable(params->tlb_enable), + configCacheEnable(params->cfg_enable), + ipaCacheEnable(params->ipa_enable), + walkCacheEnable(params->walk_enable), + tableWalkPortEnable(false), + walkCacheNonfinalEnable(params->wc_nonfinal_enable), + walkCacheS1Levels(params->wc_s1_levels), + walkCacheS2Levels(params->wc_s2_levels), + masterPortWidth(params->master_port_width), + tlbSem(params->tlb_slots), + ifcSmmuSem(1), + smmuIfcSem(1), + configSem(params->cfg_slots), + ipaSem(params->ipa_slots), + walkSem(params->walk_slots), + masterPortSem(1), + transSem(params->xlate_slots), + ptwSem(params->ptw_slots), + cycleSem(1), + tlbLat(params->tlb_lat), + ifcSmmuLat(params->ifc_smmu_lat), + smmuIfcLat(params->smmu_ifc_lat), + configLat(params->cfg_lat), + ipaLat(params->ipa_lat), + walkLat(params->walk_lat), + slaveInterfaces(params->slave_interfaces), + commandExecutor(name() + ".cmd_exec", *this), + regsMap(params->reg_map), + processCommandsEvent(this) +{ + fatal_if(regsMap.size() != SMMU_REG_SIZE, + "Invalid register map size: %#x different than SMMU_REG_SIZE = %#x\n", + regsMap.size(), SMMU_REG_SIZE); + + // Init smmu registers to 0 + memset(®s, 0, sizeof(regs)); + + // Setup RO ID registers + regs.idr0 = params->smmu_idr0; + regs.idr1 = params->smmu_idr1; + regs.idr2 = params->smmu_idr2; + regs.idr3 = params->smmu_idr3; + regs.idr4 = params->smmu_idr4; + regs.idr5 = params->smmu_idr5; + regs.iidr = params->smmu_iidr; + regs.aidr = params->smmu_aidr; + + // TODO: At the moment it possible to set the ID registers to hold + // any possible value. It would be nice to have a sanity check here + // at construction time in case some idx registers are programmed to + // store an unallowed values or if the are configuration conflicts. + warn("SMMUv3 IDx register values unchecked\n"); + + for (auto ifc : slaveInterfaces) + ifc->setSMMU(this); +} + +bool +SMMUv3::masterRecvTimingResp(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "[t] master resp addr=%#x size=%#x\n", + pkt->getAddr(), pkt->getSize()); + + // @todo: We need to pay for this and not just zero it out + pkt->headerDelay = pkt->payloadDelay = 0; + + SMMUProcess *proc = + safe_cast(pkt->popSenderState()); + + runProcessTiming(proc, pkt); + + return true; +} + +void +SMMUv3::masterRecvReqRetry() +{ + assert(!packetsToRetry.empty()); + + while (!packetsToRetry.empty()) { + SMMUAction a = packetsToRetry.front(); + + assert(a.type==ACTION_SEND_REQ || a.type==ACTION_SEND_REQ_FINAL); + + DPRINTF(SMMUv3, "[t] master retr addr=%#x size=%#x\n", + a.pkt->getAddr(), a.pkt->getSize()); + + if (!masterPort.sendTimingReq(a.pkt)) + break; + + packetsToRetry.pop(); + + /* + * ACTION_SEND_REQ_FINAL means that we have just forwarded the packet + * on the master interface; this means that we no longer hold on to + * that transaction and therefore can accept a new one. + * If the slave port was stalled then unstall it (send retry). + */ + if (a.type == ACTION_SEND_REQ_FINAL) + scheduleSlaveRetries(); + } +} + +bool +SMMUv3::masterTableWalkRecvTimingResp(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "[t] master HWTW resp addr=%#x size=%#x\n", + pkt->getAddr(), pkt->getSize()); + + // @todo: We need to pay for this and not just zero it out + pkt->headerDelay = pkt->payloadDelay = 0; + + SMMUProcess *proc = + safe_cast(pkt->popSenderState()); + + runProcessTiming(proc, pkt); + + return true; +} + +void +SMMUv3::masterTableWalkRecvReqRetry() +{ + assert(tableWalkPortEnable); + assert(!packetsTableWalkToRetry.empty()); + + while (!packetsTableWalkToRetry.empty()) { + SMMUAction a = packetsTableWalkToRetry.front(); + + assert(a.type==ACTION_SEND_REQ); + + DPRINTF(SMMUv3, "[t] master HWTW retr addr=%#x size=%#x\n", + a.pkt->getAddr(), a.pkt->getSize()); + + if (!masterTableWalkPort.sendTimingReq(a.pkt)) + break; + + packetsTableWalkToRetry.pop(); + } +} + +void +SMMUv3::scheduleSlaveRetries() +{ + for (auto ifc : slaveInterfaces) { + ifc->scheduleDeviceRetry(); + } +} + +SMMUAction +SMMUv3::runProcess(SMMUProcess *proc, PacketPtr pkt) +{ + if (system.isAtomicMode()) { + return runProcessAtomic(proc, pkt); + } else if (system.isTimingMode()) { + return runProcessTiming(proc, pkt); + } else { + panic("Not in timing or atomic mode!"); + } +} + +SMMUAction +SMMUv3::runProcessAtomic(SMMUProcess *proc, PacketPtr pkt) +{ + SMMUAction action; + Tick delay = 0; + bool finished = false; + + do { + action = proc->run(pkt); + + switch (action.type) { + case ACTION_SEND_REQ: + // Send an MMU initiated request on the table walk port if it is + // enabled. Otherwise, fall through and handle same as the final + // ACTION_SEND_REQ_FINAL request. + if (tableWalkPortEnable) { + delay += masterTableWalkPort.sendAtomic(action.pkt); + pkt = action.pkt; + break; + } + M5_FALLTHROUGH; + case ACTION_SEND_REQ_FINAL: + delay += masterPort.sendAtomic(action.pkt); + pkt = action.pkt; + break; + + case ACTION_SEND_RESP: + case ACTION_SEND_RESP_ATS: + case ACTION_SLEEP: + finished = true; + break; + + case ACTION_DELAY: + delay += action.delay; + break; + + case ACTION_TERMINATE: + panic("ACTION_TERMINATE in atomic mode\n"); + + default: + panic("Unknown action\n"); + } + } while (!finished); + + action.delay = delay; + + return action; +} + +SMMUAction +SMMUv3::runProcessTiming(SMMUProcess *proc, PacketPtr pkt) +{ + SMMUAction action = proc->run(pkt); + + switch (action.type) { + case ACTION_SEND_REQ: + // Send an MMU initiated request on the table walk port if it is + // enabled. Otherwise, fall through and handle same as the final + // ACTION_SEND_REQ_FINAL request. + if (tableWalkPortEnable) { + action.pkt->pushSenderState(proc); + + DPRINTF(SMMUv3, "[t] master HWTW req addr=%#x size=%#x\n", + action.pkt->getAddr(), action.pkt->getSize()); + + if (packetsTableWalkToRetry.empty() + && masterTableWalkPort.sendTimingReq(action.pkt)) { + scheduleSlaveRetries(); + } else { + DPRINTF(SMMUv3, "[t] master HWTW req needs retry," + " qlen=%d\n", packetsTableWalkToRetry.size()); + packetsTableWalkToRetry.push(action); + } + + break; + } + M5_FALLTHROUGH; + case ACTION_SEND_REQ_FINAL: + action.pkt->pushSenderState(proc); + + DPRINTF(SMMUv3, "[t] master req addr=%#x size=%#x\n", + action.pkt->getAddr(), action.pkt->getSize()); + + if (packetsToRetry.empty() && masterPort.sendTimingReq(action.pkt)) { + scheduleSlaveRetries(); + } else { + DPRINTF(SMMUv3, "[t] master req needs retry, qlen=%d\n", + packetsToRetry.size()); + packetsToRetry.push(action); + } + + break; + + case ACTION_SEND_RESP: + // @todo: We need to pay for this and not just zero it out + action.pkt->headerDelay = action.pkt->payloadDelay = 0; + + DPRINTF(SMMUv3, "[t] slave resp addr=%#x size=%#x\n", + action.pkt->getAddr(), + action.pkt->getSize()); + + assert(action.ifc); + action.ifc->schedTimingResp(action.pkt); + + delete proc; + break; + + case ACTION_SEND_RESP_ATS: + // @todo: We need to pay for this and not just zero it out + action.pkt->headerDelay = action.pkt->payloadDelay = 0; + + DPRINTF(SMMUv3, "[t] ATS slave resp addr=%#x size=%#x\n", + action.pkt->getAddr(), action.pkt->getSize()); + + assert(action.ifc); + action.ifc->schedAtsTimingResp(action.pkt); + + delete proc; + break; + + case ACTION_DELAY: + case ACTION_SLEEP: + break; + + case ACTION_TERMINATE: + delete proc; + break; + + default: + panic("Unknown action\n"); + } + + return action; +} + +void +SMMUv3::processCommands() +{ + DPRINTF(SMMUv3, "processCommands()\n"); + + if (system.isAtomicMode()) { + SMMUAction a = runProcessAtomic(&commandExecutor, NULL); + (void) a; + } else if (system.isTimingMode()) { + if (!commandExecutor.isBusy()) + runProcessTiming(&commandExecutor, NULL); + } else { + panic("Not in timing or atomic mode!"); + } +} + +void +SMMUv3::processCommand(const SMMUCommand &cmd) +{ + switch (cmd.type) { + case CMD_PRF_CONFIG: + DPRINTF(SMMUv3, "CMD_PREFETCH_CONFIG - ignored\n"); + break; + + case CMD_PRF_ADDR: + DPRINTF(SMMUv3, "CMD_PREFETCH_ADDR - ignored\n"); + break; + + case CMD_INV_STE: + DPRINTF(SMMUv3, "CMD_INV_STE sid=%#x\n", cmd.data[0]); + configCache.invalidateSID(cmd.data[0]); + break; + + case CMD_INV_CD: + DPRINTF(SMMUv3, "CMD_INV_CD sid=%#x ssid=%#x\n", + cmd.data[0], cmd.data[1]); + configCache.invalidateSSID(cmd.data[0], cmd.data[1]); + break; + + case CMD_INV_CD_ALL: + DPRINTF(SMMUv3, "CMD_INV_CD_ALL sid=%#x\n", cmd.data[0]); + configCache.invalidateSID(cmd.data[0]); + break; + + case CMD_INV_ALL: + DPRINTF(SMMUv3, "CMD_INV_ALL\n"); + configCache.invalidateAll(); + break; + + case CMD_TLBI_ALL: + DPRINTF(SMMUv3, "CMD_TLBI_ALL\n"); + for (auto slave_interface : slaveInterfaces) { + slave_interface->microTLB->invalidateAll(); + slave_interface->mainTLB->invalidateAll(); + } + tlb.invalidateAll(); + ipaCache.invalidateAll(); + walkCache.invalidateAll(); + break; + + case CMD_TLBI_ASID: + DPRINTF(SMMUv3, "CMD_TLBI_ASID asid=%#x vmid=%#x\n", + cmd.data[0], cmd.data[1]); + for (auto slave_interface : slaveInterfaces) { + slave_interface->microTLB->invalidateASID( + cmd.data[0], cmd.data[1]); + slave_interface->mainTLB->invalidateASID( + cmd.data[0], cmd.data[1]); + } + tlb.invalidateASID(cmd.data[0], cmd.data[1]); + walkCache.invalidateASID(cmd.data[0], cmd.data[1]); + break; + + case CMD_TLBI_VAAL: + DPRINTF(SMMUv3, "CMD_TLBI_VAAL va=%#08x vmid=%#x\n", + cmd.data[0], cmd.data[1]); + for (auto slave_interface : slaveInterfaces) { + slave_interface->microTLB->invalidateVAA( + cmd.data[0], cmd.data[1]); + slave_interface->mainTLB->invalidateVAA( + cmd.data[0], cmd.data[1]); + } + tlb.invalidateVAA(cmd.data[0], cmd.data[1]); + break; + + case CMD_TLBI_VAA: + DPRINTF(SMMUv3, "CMD_TLBI_VAA va=%#08x vmid=%#x\n", + cmd.data[0], cmd.data[1]); + for (auto slave_interface : slaveInterfaces) { + slave_interface->microTLB->invalidateVAA( + cmd.data[0], cmd.data[1]); + slave_interface->mainTLB->invalidateVAA( + cmd.data[0], cmd.data[1]); + } + tlb.invalidateVAA(cmd.data[0], cmd.data[1]); + walkCache.invalidateVAA(cmd.data[0], cmd.data[1]); + break; + + case CMD_TLBI_VAL: + DPRINTF(SMMUv3, "CMD_TLBI_VAL va=%#08x asid=%#x vmid=%#x\n", + cmd.data[0], cmd.data[1], cmd.data[2]); + for (auto slave_interface : slaveInterfaces) { + slave_interface->microTLB->invalidateVA( + cmd.data[0], cmd.data[1], cmd.data[2]); + slave_interface->mainTLB->invalidateVA( + cmd.data[0], cmd.data[1], cmd.data[2]); + } + tlb.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]); + break; + + case CMD_TLBI_VA: + DPRINTF(SMMUv3, "CMD_TLBI_VA va=%#08x asid=%#x vmid=%#x\n", + cmd.data[0], cmd.data[1], cmd.data[2]); + for (auto slave_interface : slaveInterfaces) { + slave_interface->microTLB->invalidateVA( + cmd.data[0], cmd.data[1], cmd.data[2]); + slave_interface->mainTLB->invalidateVA( + cmd.data[0], cmd.data[1], cmd.data[2]); + } + tlb.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]); + walkCache.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]); + break; + + case CMD_TLBI_VM_IPAL: + DPRINTF(SMMUv3, "CMD_TLBI_VM_IPAL ipa=%#08x vmid=%#x\n", + cmd.data[0], cmd.data[1]); + // This does not invalidate TLBs containing + // combined Stage1 + Stage2 translations, as per the spec. + ipaCache.invalidateIPA(cmd.data[0], cmd.data[1]); + walkCache.invalidateVMID(cmd.data[1]); + break; + + case CMD_TLBI_VM_IPA: + DPRINTF(SMMUv3, "CMD_TLBI_VM_IPA ipa=%#08x vmid=%#x\n", + cmd.data[0], cmd.data[1]); + // This does not invalidate TLBs containing + // combined Stage1 + Stage2 translations, as per the spec. + ipaCache.invalidateIPA(cmd.data[0], cmd.data[1]); + walkCache.invalidateVMID(cmd.data[1]); + break; + + case CMD_TLBI_VM_S12: + DPRINTF(SMMUv3, "CMD_TLBI_VM_S12 vmid=%#x\n", cmd.data[0]); + for (auto slave_interface : slaveInterfaces) { + slave_interface->microTLB->invalidateVMID(cmd.data[0]); + slave_interface->mainTLB->invalidateVMID(cmd.data[0]); + } + tlb.invalidateVMID(cmd.data[0]); + ipaCache.invalidateVMID(cmd.data[0]); + walkCache.invalidateVMID(cmd.data[0]); + break; + + case CMD_RESUME_S: + DPRINTF(SMMUv3, "CMD_RESUME_S\n"); + panic("resume unimplemented"); + break; + + default: + warn("Unimplemented command %#x\n", cmd.type); + break; + } +} + +const PageTableOps* +SMMUv3::getPageTableOps(uint8_t trans_granule) +{ + static V8PageTableOps4k ptOps4k; + static V8PageTableOps64k ptOps64k; + + switch (trans_granule) { + case TRANS_GRANULE_4K: return &ptOps4k; + case TRANS_GRANULE_64K: return &ptOps64k; + default: + panic("Unknown translation granule size %d", trans_granule); + } +} + +Tick +SMMUv3::readControl(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "readControl: addr=%08x size=%d\n", + pkt->getAddr(), pkt->getSize()); + + int offset = pkt->getAddr() - regsMap.start(); + assert(offset >= 0 && offset < SMMU_REG_SIZE); + + if (inSecureBlock(offset)) { + warn("smmu: secure registers (0x%x) are not implemented\n", + offset); + } + + auto reg_ptr = regs.data + offset; + + switch (pkt->getSize()) { + case sizeof(uint32_t): + pkt->setLE(*reinterpret_cast(reg_ptr)); + break; + case sizeof(uint64_t): + pkt->setLE(*reinterpret_cast(reg_ptr)); + break; + default: + panic("smmu: unallowed access size: %d bytes\n", pkt->getSize()); + break; + } + + pkt->makeAtomicResponse(); + + return 0; +} + +Tick +SMMUv3::writeControl(PacketPtr pkt) +{ + int offset = pkt->getAddr() - regsMap.start(); + assert(offset >= 0 && offset < SMMU_REG_SIZE); + + DPRINTF(SMMUv3, "writeControl: addr=%08x size=%d data=%16x\n", + pkt->getAddr(), pkt->getSize(), + pkt->getSize() == sizeof(uint64_t) ? + pkt->getLE() : pkt->getLE()); + + switch (offset) { + case offsetof(SMMURegs, cr0): + assert(pkt->getSize() == sizeof(uint32_t)); + regs.cr0 = regs.cr0ack = pkt->getLE(); + break; + + case offsetof(SMMURegs, cr1): + case offsetof(SMMURegs, cr2): + case offsetof(SMMURegs, strtab_base_cfg): + case offsetof(SMMURegs, eventq_cons): + case offsetof(SMMURegs, eventq_irq_cfg1): + case offsetof(SMMURegs, priq_cons): + assert(pkt->getSize() == sizeof(uint32_t)); + *reinterpret_cast(regs.data + offset) = + pkt->getLE(); + break; + + case offsetof(SMMURegs, cmdq_prod): + assert(pkt->getSize() == sizeof(uint32_t)); + *reinterpret_cast(regs.data + offset) = + pkt->getLE(); + schedule(processCommandsEvent, nextCycle()); + break; + + case offsetof(SMMURegs, strtab_base): + case offsetof(SMMURegs, eventq_irq_cfg0): + assert(pkt->getSize() == sizeof(uint64_t)); + *reinterpret_cast(regs.data + offset) = + pkt->getLE(); + break; + + case offsetof(SMMURegs, cmdq_base): + assert(pkt->getSize() == sizeof(uint64_t)); + *reinterpret_cast(regs.data + offset) = + pkt->getLE(); + regs.cmdq_cons = 0; + regs.cmdq_prod = 0; + break; + + + case offsetof(SMMURegs, eventq_base): + assert(pkt->getSize() == sizeof(uint64_t)); + *reinterpret_cast(regs.data + offset) = + pkt->getLE(); + regs.eventq_cons = 0; + regs.eventq_prod = 0; + break; + + case offsetof(SMMURegs, priq_base): + assert(pkt->getSize() == sizeof(uint64_t)); + *reinterpret_cast(regs.data + offset) = + pkt->getLE(); + regs.priq_cons = 0; + regs.priq_prod = 0; + break; + + default: + if (inSecureBlock(offset)) { + warn("smmu: secure registers (0x%x) are not implemented\n", + offset); + } else { + warn("smmu: write to read-only/undefined register at 0x%x\n", + offset); + } + } + + pkt->makeAtomicResponse(); + + return 0; +} + +bool +SMMUv3::inSecureBlock(uint32_t offs) const +{ + if (offs >= offsetof(SMMURegs, _secure_regs) && offs < SMMU_SECURE_SZ) + return true; + else + return false; +} + +void +SMMUv3::init() +{ + // make sure both sides are connected and have the same block size + if (!masterPort.isConnected()) + fatal("Master port is not connected.\n"); + + // If the second master port is connected for the table walks, enable + // the mode to send table walks through this port instead + if (masterTableWalkPort.isConnected()) + tableWalkPortEnable = true; + + // notify the master side of our address ranges + for (auto ifc : slaveInterfaces) { + ifc->sendRange(); + } + + if (controlPort.isConnected()) + controlPort.sendRangeChange(); +} + +void +SMMUv3::regStats() +{ + MemObject::regStats(); + + using namespace Stats; + + for (size_t i = 0; i < slaveInterfaces.size(); i++) { + slaveInterfaces[i]->microTLB->regStats( + csprintf("%s.utlb%d", name(), i)); + slaveInterfaces[i]->mainTLB->regStats( + csprintf("%s.maintlb%d", name(), i)); + } + + tlb.regStats(name() + ".tlb"); + configCache.regStats(name() + ".cfg"); + ipaCache.regStats(name() + ".ipa"); + walkCache.regStats(name() + ".walk"); + + steL1Fetches + .name(name() + ".steL1Fetches") + .desc("STE L1 fetches") + .flags(pdf); + + steFetches + .name(name() + ".steFetches") + .desc("STE fetches") + .flags(pdf); + + cdL1Fetches + .name(name() + ".cdL1Fetches") + .desc("CD L1 fetches") + .flags(pdf); + + cdFetches + .name(name() + ".cdFetches") + .desc("CD fetches") + .flags(pdf); + + translationTimeDist + .init(0, 2000000, 2000) + .name(name() + ".translationTimeDist") + .desc("Time to translate address") + .flags(pdf); + + ptwTimeDist + .init(0, 2000000, 2000) + .name(name() + ".ptwTimeDist") + .desc("Time to walk page tables") + .flags(pdf); +} + +DrainState +SMMUv3::drain() +{ + panic("SMMUv3 doesn't support draining\n"); +} + +void +SMMUv3::serialize(CheckpointOut &cp) const +{ + DPRINTF(Checkpoint, "Serializing SMMUv3\n"); + + SERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0])); +} + +void +SMMUv3::unserialize(CheckpointIn &cp) +{ + DPRINTF(Checkpoint, "Unserializing SMMUv3\n"); + + UNSERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0])); +} + +Port& +SMMUv3::getPort(const std::string &name, PortID id) +{ + if (name == "master") { + return masterPort; + } else if (name == "master_walker") { + return masterTableWalkPort; + } else if (name == "control") { + return controlPort; + } else { + return MemObject::getPort(name, id); + } +} + +SMMUv3* +SMMUv3Params::create() +{ + return new SMMUv3(this); +} diff --git a/src/dev/arm/smmu_v3.hh b/src/dev/arm/smmu_v3.hh new file mode 100644 index 000000000..f02ef2f77 --- /dev/null +++ b/src/dev/arm/smmu_v3.hh @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_HH__ +#define __DEV_ARM_SMMU_V3_HH__ + +#include +#include +#include +#include +#include + +#include "base/statistics.hh" +#include "dev/arm/smmu_v3_caches.hh" +#include "dev/arm/smmu_v3_cmdexec.hh" +#include "dev/arm/smmu_v3_defs.hh" +#include "dev/arm/smmu_v3_events.hh" +#include "dev/arm/smmu_v3_ports.hh" +#include "dev/arm/smmu_v3_proc.hh" +#include "dev/arm/smmu_v3_ptops.hh" +#include "dev/arm/smmu_v3_slaveifc.hh" +#include "mem/mem_object.hh" +#include "mem/packet.hh" +#include "params/SMMUv3.hh" +#include "sim/eventq.hh" + +/** + * @file: + * This is an implementation of the SMMUv3 architecture. + * + * What can it do? + * - Single-stage and nested translation with 4k or 64k granule. 16k would + * be straightforward to add. + * - Large pages are supported. + * - Works with any gem5 device as long as it is issuing packets with a + * valid (Sub)StreamId + * + * What it can't do? + * - Fragment stage 1 page when the underlying stage 2 page is smaller. S1 + * page size > S2 page size is not supported + * - Invalidations take zero time. This wouldn't be hard to fix. + * - Checkpointing is not supported + * - Stall/resume for faulting transactions is not supported + */ +class SMMUTranslationProcess; + +class SMMUv3 : public MemObject +{ + protected: + + friend class SMMUProcess; + friend class SMMUTranslationProcess; + friend class SMMUCommandExecProcess; + friend class SMMUv3SlaveInterface; + + const System &system; + const MasterID masterId; + + SMMUMasterPort masterPort; + SMMUMasterTableWalkPort masterTableWalkPort; + SMMUControlPort controlPort; + + ARMArchTLB tlb; + ConfigCache configCache; + IPACache ipaCache; + WalkCache walkCache; + + const bool tlbEnable; + const bool configCacheEnable; + const bool ipaCacheEnable; + const bool walkCacheEnable; + bool tableWalkPortEnable; + + const bool walkCacheNonfinalEnable; + const unsigned walkCacheS1Levels; + const unsigned walkCacheS2Levels; + const unsigned masterPortWidth; // in bytes + + SMMUSemaphore tlbSem; + SMMUSemaphore ifcSmmuSem; + SMMUSemaphore smmuIfcSem; + SMMUSemaphore configSem; + SMMUSemaphore ipaSem; + SMMUSemaphore walkSem; + SMMUSemaphore masterPortSem; + + SMMUSemaphore transSem; // max N transactions in SMMU + SMMUSemaphore ptwSem; // max N concurrent PTWs + SMMUSemaphore cycleSem; // max 1 table walk per cycle + + // Timing parameters + const Cycles tlbLat; + const Cycles ifcSmmuLat; + const Cycles smmuIfcLat; + const Cycles configLat; + const Cycles ipaLat; + const Cycles walkLat; + + // Stats + Stats::Scalar steL1Fetches; + Stats::Scalar steFetches; + Stats::Scalar cdL1Fetches; + Stats::Scalar cdFetches; + Stats::Distribution translationTimeDist; + Stats::Distribution ptwTimeDist; + + std::vector slaveInterfaces; + + SMMUCommandExecProcess commandExecutor; + + const AddrRange regsMap; + SMMURegs regs; + + bool inSecureBlock(uint32_t offs) const; + + std::queue packetsToRetry; + std::queue packetsTableWalkToRetry; + + + void scheduleSlaveRetries(); + + SMMUAction runProcess(SMMUProcess *proc, PacketPtr pkt); + SMMUAction runProcessAtomic(SMMUProcess *proc, PacketPtr pkt); + SMMUAction runProcessTiming(SMMUProcess *proc, PacketPtr pkt); + + void processCommands(); + EventWrapper processCommandsEvent; + + void processCommand(const SMMUCommand &cmd); + + const PageTableOps *getPageTableOps(uint8_t trans_granule); + + public: + SMMUv3(SMMUv3Params *p); + virtual ~SMMUv3() {} + + virtual void init() override; + virtual void regStats() override; + + Tick slaveRecvAtomic(PacketPtr pkt, PortID id); + bool slaveRecvTimingReq(PacketPtr pkt, PortID id); + bool masterRecvTimingResp(PacketPtr pkt); + void masterRecvReqRetry(); + + bool masterTableWalkRecvTimingResp(PacketPtr pkt); + void masterTableWalkRecvReqRetry(); + + Tick readControl(PacketPtr pkt); + Tick writeControl(PacketPtr pkt); + + DrainState drain() override; + void serialize(CheckpointOut &cp) const override; + void unserialize(CheckpointIn &cp) override; + + virtual Port &getPort(const std::string &name, + PortID id = InvalidPortID) override; +}; + +#endif /* __DEV_ARM_SMMU_V3_HH__ */ diff --git a/src/dev/arm/smmu_v3_caches.cc b/src/dev/arm/smmu_v3_caches.cc new file mode 100644 index 000000000..6dcaec6b6 --- /dev/null +++ b/src/dev/arm/smmu_v3_caches.cc @@ -0,0 +1,1226 @@ +/* + * Copyright (c) 2014, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + * Damian Richardson + */ + +#include "dev/arm/smmu_v3_caches.hh" + +#include + +#include "base/bitfield.hh" +#include "base/intmath.hh" +#include "base/logging.hh" +#include "sim/stats.hh" + + +// taken from hex expansion of pi +#define SMMUTLB_SEED 0xEA752DFE +#define ARMARCHTLB_SEED 0x8B021FA1 +#define IPACACHE_SEED 0xE5A0CC0F +#define CONFIGCACHE_SEED 0xB56F74E8 +#define WALKCACHE_SEED 0x18ACF3D6 + +/* + * BaseCache + * + * TODO: move more code into this base class to reduce duplication. + */ + +SMMUv3BaseCache::SMMUv3BaseCache(const std::string &policy_name, uint32_t seed) : + replacementPolicy(decodePolicyName(policy_name)), + nextToReplace(0), + random(seed), + useStamp(0) +{} + +int +SMMUv3BaseCache::decodePolicyName(const std::string &policy_name) +{ + if (policy_name == "rr") { + return SMMU_CACHE_REPL_ROUND_ROBIN; + } else if (policy_name == "rand") { + return SMMU_CACHE_REPL_RANDOM; + } else if (policy_name == "lru") { + return SMMU_CACHE_REPL_LRU; + } else { + panic("Unknown cache replacement policy '%s'\n", policy_name); + } +} + +void +SMMUv3BaseCache::regStats(const std::string &name) +{ + using namespace Stats; + + + averageLookups + .name(name + ".averageLookups") + .desc("Average number lookups per second") + .flags(pdf); + + totalLookups + .name(name + ".totalLookups") + .desc("Total number of lookups") + .flags(pdf); + + averageLookups = totalLookups / simSeconds; + + + averageMisses + .name(name + ".averageMisses") + .desc("Average number misses per second") + .flags(pdf); + + totalMisses + .name(name + ".totalMisses") + .desc("Total number of misses") + .flags(pdf); + + averageMisses = totalMisses / simSeconds; + + + averageUpdates + .name(name + ".averageUpdates") + .desc("Average number updates per second") + .flags(pdf); + + totalUpdates + .name(name + ".totalUpdates") + .desc("Total number of updates") + .flags(pdf); + + averageUpdates = totalUpdates / simSeconds; + + + averageHitRate + .name(name + ".averageHitRate") + .desc("Average hit rate") + .flags(pdf); + + averageHitRate = (totalLookups - totalMisses) / totalLookups; + + insertions + .name(name + ".insertions") + .desc("Number of insertions (not replacements)") + .flags(pdf); +} + + + +/* + * SMMUTLB + */ + +SMMUTLB::SMMUTLB(unsigned numEntries, unsigned _associativity, + const std::string &policy) +: + SMMUv3BaseCache(policy, SMMUTLB_SEED), + associativity(_associativity) +{ + if (associativity == 0) + associativity = numEntries; // fully associative + + if (numEntries == 0) + fatal("SMMUTLB must have at least one entry\n"); + + if (associativity > numEntries) + fatal("SMMUTLB associativity cannot be higher than " + "its number of entries\n"); + + unsigned num_sets = numEntries / associativity; + + if (num_sets*associativity != numEntries) + fatal("Number of SMMUTLB entries must be divisible " + "by its associativity\n"); + + Entry e; + e.valid = false; + + Set set(associativity, e); + sets.resize(num_sets, set); +} + +const SMMUTLB::Entry* +SMMUTLB::lookup(uint32_t sid, uint32_t ssid, + Addr va, bool updStats) +{ + const Entry *result = NULL; + + Set &set = sets[pickSetIdx(va)]; + + for (size_t i = 0; i < set.size(); i++) { + const Entry &e = set[i]; + + if (e.valid && (e.va & e.vaMask) == (va & e.vaMask) && + e.sid==sid && e.ssid==ssid) + { + if (result != NULL) + panic("SMMUTLB: duplicate entry found!\n"); + + result = &e; + break; + } + } + + if (updStats) { + if (result) + result->lastUsed = useStamp++; + + totalLookups++; + if (result == NULL) + totalMisses++; + } + + return result; +} + +const SMMUTLB::Entry* +SMMUTLB::lookupAnyVA(uint32_t sid, uint32_t ssid, bool updStats) +{ + const Entry *result = NULL; + + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + const Entry &e = set[i]; + + if (e.valid && e.sid==sid && e.ssid==ssid) { + result = &e; + break; + } + } + } + + if (updStats) { + totalLookups++; + if (result == NULL) + totalMisses++; + } + + return result; +} + +void +SMMUTLB::store(const Entry &incoming, AllocPolicy alloc) +{ + if (!incoming.valid) + panic("Tried to store an invalid entry\n"); + + incoming.lastUsed = 0; + + const Entry *existing = + lookup(incoming.sid, incoming.ssid, incoming.va, false); + + if (existing) { + *const_cast (existing) = incoming; + } else { + Set &set = sets[pickSetIdx(incoming.va)]; + set[pickEntryIdxToReplace(set, alloc)] = incoming; + } + + totalUpdates++; +} + +void +SMMUTLB::invalidateVA(Addr va, uint16_t asid, uint16_t vmid) +{ + Set &set = sets[pickSetIdx(va)]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if ((e.va & e.vaMask) == (va & e.vaMask) && + e.asid==asid && e.vmid==vmid) + { + e.valid = false; + } + } +} + +void +SMMUTLB::invalidateVAA(Addr va, uint16_t vmid) +{ + Set &set = sets[pickSetIdx(va)]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if ((e.va & e.vaMask) == (va & e.vaMask) && e.vmid==vmid) + e.valid = false; + } +} + +void +SMMUTLB::invalidateASID(uint16_t asid, uint16_t vmid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.asid==asid && e.vmid==vmid) + e.valid = false; + } + } +} + +void +SMMUTLB::invalidateVMID(uint16_t vmid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.vmid == vmid) + e.valid = false; + } + } +} + +void +SMMUTLB::invalidateAll() +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) + set[i].valid = false; + } +} + +size_t +SMMUTLB::pickSetIdx(Addr va) const +{ + return (va >> 12) % sets.size(); +} + +size_t +SMMUTLB::pickEntryIdxToReplace(const Set &set, AllocPolicy alloc) +{ + if (alloc == ALLOC_LAST_WAY) + return associativity - 1; + + uint32_t lru_tick = UINT32_MAX; + size_t lru_idx = 0; + size_t max_idx = + alloc==ALLOC_ANY_BUT_LAST_WAY ? + set.size()-1 : set.size(); + + for (size_t i = 0; i < max_idx; i++) { + if (!set[i].valid) { + insertions++; + return i; + } + + if (set[i].lastUsed < lru_tick) { + lru_idx = i; + lru_tick = set[i].lastUsed; + } + } + + switch (replacementPolicy) { + case SMMU_CACHE_REPL_ROUND_ROBIN: + switch (alloc) { + case ALLOC_ANY_WAY: + return nextToReplace = ((nextToReplace+1) % associativity); + case ALLOC_ANY_BUT_LAST_WAY: + return nextToReplace = ((nextToReplace+1) % (associativity-1)); + default: + panic("Unknown allocation mode %d\n", alloc); + } + + case SMMU_CACHE_REPL_RANDOM: + switch (alloc) { + case ALLOC_ANY_WAY: + return random.random(0, associativity-1); + case ALLOC_ANY_BUT_LAST_WAY: + return random.random(0, associativity-2); + default: + panic("Unknown allocation mode %d\n", alloc); + } + + case SMMU_CACHE_REPL_LRU: + return lru_idx; + + default: + panic("Unknown replacement policy %d\n", replacementPolicy); + } +} + + + +/* + * ARMArchTLB + */ + +ARMArchTLB::ARMArchTLB(unsigned numEntries, unsigned _associativity, + const std::string &policy) +: + SMMUv3BaseCache(policy, ARMARCHTLB_SEED), + associativity(_associativity) +{ + if (associativity == 0) + associativity = numEntries; // fully associative + + if (numEntries == 0) + fatal("ARMArchTLB must have at least one entry\n"); + + if (associativity > numEntries) + fatal("ARMArchTLB associativity cannot be higher than " + "its number of entries\n"); + + unsigned num_sets = numEntries / associativity; + + if (num_sets*associativity != numEntries) + fatal("Number of ARMArchTLB entries must be divisible " + "by its associativity\n"); + + Entry e; + e.valid = false; + + Set set(associativity, e); + sets.resize(num_sets, set); +} + +const ARMArchTLB::Entry * +ARMArchTLB::lookup(Addr va, uint16_t asid, uint16_t vmid, bool updStats) +{ + const Entry *result = NULL; + + Set &set = sets[pickSetIdx(va, asid, vmid)]; + + for (size_t i = 0; i < set.size(); i++) { + const Entry &e = set[i]; + + if (e.valid && (e.va & e.vaMask) == (va & e.vaMask) && + e.asid==asid && e.vmid==vmid) + { + if (result != NULL) + panic("ARMArchTLB: duplicate entry found!\n"); + + result = &e; + break; + } + } + + if (updStats) { + if (result) + result->lastUsed = useStamp++; + + totalLookups++; + if (result == NULL) + totalMisses++; + } + + return result; +} + +void +ARMArchTLB::store(const Entry &incoming) +{ + if (!incoming.valid) + panic("Tried to store an invalid entry\n"); + + incoming.lastUsed = 0; + + const Entry *existing = + lookup(incoming.va, incoming.asid, incoming.vmid, false); + + if (existing) { + *const_cast (existing) = incoming; + } else { + Set &set = sets[pickSetIdx(incoming.va, incoming.asid, incoming.vmid)]; + set[pickEntryIdxToReplace(set)] = incoming; + } + + totalUpdates++; +} + +void +ARMArchTLB::invalidateVA(Addr va, uint16_t asid, uint16_t vmid) +{ + Set &set = sets[pickSetIdx(va, asid, vmid)]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if ((e.va & e.vaMask) == (va & e.vaMask) && + e.asid==asid && e.vmid==vmid) + { + e.valid = false; + } + } +} + +void +ARMArchTLB::invalidateVAA(Addr va, uint16_t vmid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if ((e.va & e.vaMask) == (va & e.vaMask) && e.vmid==vmid) + e.valid = false; + } + } +} + +void +ARMArchTLB::invalidateASID(uint16_t asid, uint16_t vmid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.asid==asid && e.vmid==vmid) + e.valid = false; + } + } +} + +void +ARMArchTLB::invalidateVMID(uint16_t vmid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.vmid == vmid) + e.valid = false; + } + } +} + +void +ARMArchTLB::invalidateAll() +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) + set[i].valid = false; + } +} + +size_t +ARMArchTLB::pickSetIdx(Addr va, uint16_t asid, uint16_t vmid) const +{ + return ((va >> 12) ^ asid ^ vmid) % sets.size(); +} + +size_t +ARMArchTLB::pickEntryIdxToReplace(const Set &set) +{ + size_t lru_idx = 0; + uint32_t lru_tick = UINT32_MAX; + + for (size_t i = 0; i < set.size(); i++) { + if (!set[i].valid) { + insertions++; + return i; + } + + if (set[i].lastUsed < lru_tick) { + lru_idx = i; + lru_tick = set[i].lastUsed; + } + } + + switch (replacementPolicy) { + case SMMU_CACHE_REPL_ROUND_ROBIN: + return nextToReplace = ((nextToReplace+1) % associativity); + + case SMMU_CACHE_REPL_RANDOM: + return random.random(0, associativity-1); + + case SMMU_CACHE_REPL_LRU: + return lru_idx; + + default: + panic("Unknown replacement policy %d\n", replacementPolicy); + } + +} + +/* + * IPACache + */ + +IPACache::IPACache(unsigned numEntries, unsigned _associativity, + const std::string &policy) +: + SMMUv3BaseCache(policy, IPACACHE_SEED), + associativity(_associativity) +{ + if (associativity == 0) + associativity = numEntries; // fully associative + + if (numEntries == 0) + fatal("IPACache must have at least one entry\n"); + + if (associativity > numEntries) + fatal("IPACache associativity cannot be higher than " + "its number of entries\n"); + + unsigned num_sets = numEntries / associativity; + + if (num_sets*associativity != numEntries) + fatal("Number of IPACache entries must be divisible " + "by its associativity\n"); + + Entry e; + e.valid = false; + + Set set(associativity, e); + sets.resize(num_sets, set); +} + +const IPACache::Entry* +IPACache::lookup(Addr ipa, uint16_t vmid, bool updStats) +{ + const Entry *result = NULL; + + Set &set = sets[pickSetIdx(ipa, vmid)]; + + for (size_t i = 0; i < set.size(); i++) { + const Entry &e = set[i]; + + if (e.valid && (e.ipa & e.ipaMask) == (ipa & e.ipaMask) && + e.vmid==vmid) + { + if (result != NULL) + panic("IPACache: duplicate entry found!\n"); + + result = &e; + break; + } + } + + if (updStats) { + if (result) + result->lastUsed = useStamp++; + + totalLookups++; + if (result == NULL) + totalMisses++; + } + + return result; +} + +void +IPACache::store(const Entry &incoming) +{ + if (!incoming.valid) + panic("Tried to store an invalid entry\n"); + + incoming.lastUsed = 0; + + const Entry *existing = lookup(incoming.ipa, incoming.vmid, false); + + if (existing) { + *const_cast (existing) = incoming; + } else { + Set &set = sets[pickSetIdx(incoming.ipa, incoming.vmid)]; + set[pickEntryIdxToReplace(set)] = incoming; + } + + totalUpdates++; +} + +void +IPACache::invalidateIPA(Addr ipa, uint16_t vmid) +{ + Set &set = sets[pickSetIdx(ipa, vmid)]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if ((e.ipa & e.ipaMask) == (ipa & e.ipaMask) && e.vmid==vmid) + e.valid = false; + } +} + +void +IPACache::invalidateIPAA(Addr ipa) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if ((e.ipa & e.ipaMask) == (ipa & e.ipaMask)) + e.valid = false; + } + } +} + +void +IPACache::invalidateVMID(uint16_t vmid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.vmid == vmid) + e.valid = false; + } + } +} + +void +IPACache::invalidateAll() +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) + set[i].valid = false; + } +} + +size_t +IPACache::pickSetIdx(Addr va, uint16_t vmid) const +{ + return ((va >> 12) ^ vmid) % sets.size(); +} + +size_t +IPACache::pickEntryIdxToReplace(const Set &set) +{ + size_t lru_idx = 0; + uint32_t lru_tick = UINT32_MAX; + + for (size_t i = 0; i < set.size(); i++) { + if (!set[i].valid) { + insertions++; + return i; + } + + if (set[i].lastUsed < lru_tick) { + lru_idx = i; + lru_tick = set[i].lastUsed; + } + } + + switch (replacementPolicy) { + case SMMU_CACHE_REPL_ROUND_ROBIN: + return nextToReplace = ((nextToReplace+1) % associativity); + + case SMMU_CACHE_REPL_RANDOM: + return random.random(0, associativity-1); + + case SMMU_CACHE_REPL_LRU: + return lru_idx; + + default: + panic("Unknown replacement policy %d\n", replacementPolicy); + } + +} + +/* + * ConfigCache + */ + +ConfigCache::ConfigCache(unsigned numEntries, unsigned _associativity, + const std::string &policy) +: + SMMUv3BaseCache(policy, CONFIGCACHE_SEED), + associativity(_associativity) +{ + if (associativity == 0) + associativity = numEntries; // fully associative + + if (numEntries == 0) + fatal("ConfigCache must have at least one entry\n"); + + if (associativity > numEntries) + fatal("ConfigCache associativity cannot be higher than " + "its number of entries\n"); + + unsigned num_sets = numEntries / associativity; + + if (num_sets*associativity != numEntries) + fatal("Number of ConfigCache entries must be divisible " + "by its associativity\n"); + + Entry e; + e.valid = false; + + Set set(associativity, e); + sets.resize(num_sets, set); +} + +const ConfigCache::Entry * +ConfigCache::lookup(uint32_t sid, uint32_t ssid, bool updStats) +{ + const Entry *result = NULL; + + Set &set = sets[pickSetIdx(sid, ssid)]; + + for (size_t i = 0; i < set.size(); i++) { + const Entry &e = set[i]; + + if (e.valid && e.sid==sid && e.ssid==ssid) + { + if (result != NULL) + panic("ConfigCache: duplicate entry found!\n"); + + result = &e; + break; + } + } + + if (updStats) { + if (result) + result->lastUsed = useStamp++; + + totalLookups++; + if (result == NULL) + totalMisses++; + } + + return result; +} + +void +ConfigCache::store(const Entry &incoming) +{ + if (!incoming.valid) + panic("Tried to store an invalid entry\n"); + + incoming.lastUsed = 0; + + const Entry *existing = lookup(incoming.sid, incoming.ssid, false); + + if (existing) { + *const_cast (existing) = incoming; + } else { + Set &set = sets[pickSetIdx(incoming.sid, incoming.ssid)]; + set[pickEntryIdxToReplace(set)] = incoming; + } + + totalUpdates++; +} + +void +ConfigCache::invalidateSSID(uint32_t sid, uint32_t ssid) +{ + Set &set = sets[pickSetIdx(sid, ssid)]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.sid==sid && e.ssid==ssid) + e.valid = false; + } +} + +void +ConfigCache::invalidateSID(uint32_t sid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.sid == sid) + e.valid = false; + } + } +} + +void +ConfigCache::invalidateAll() +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) + set[i].valid = false; + } +} + +size_t +ConfigCache::pickSetIdx(uint32_t sid, uint32_t ssid) const +{ + return (sid^ssid) % sets.size(); +} + +size_t +ConfigCache::pickEntryIdxToReplace(const Set &set) +{ + size_t lru_idx = 0; + uint32_t lru_tick = UINT32_MAX; + + for (size_t i = 0; i < set.size(); i++) { + if (!set[i].valid) { + insertions++; + return i; + } + + if (set[i].lastUsed < lru_tick) { + lru_idx = i; + lru_tick = set[i].lastUsed; + } + } + + switch (replacementPolicy) { + case SMMU_CACHE_REPL_ROUND_ROBIN: + return nextToReplace = ((nextToReplace+1) % associativity); + + case SMMU_CACHE_REPL_RANDOM: + return random.random(0, associativity-1); + + case SMMU_CACHE_REPL_LRU: + return lru_idx; + + default: + panic("Unknown replacement policy %d\n", replacementPolicy); + } + +} + +/* + * WalkCache + */ + +WalkCache::WalkCache(const std::array &_sizes, + unsigned _associativity, const std::string &policy) : + SMMUv3BaseCache(policy, WALKCACHE_SEED), + associativity(_associativity), + sizes() +{ + unsigned numEntries = std::accumulate(&_sizes[0], + &_sizes[2*WALK_CACHE_LEVELS], 0); + + if (associativity == 0) + associativity = numEntries; // fully associative + + if (numEntries == 0) + fatal("WalkCache must have at least one entry\n"); + + for (size_t i = 0; i < 2*WALK_CACHE_LEVELS; i++){ + if (_sizes[i] % associativity != 0) + fatal("Number of WalkCache entries at each level must be " + "divisible by WalkCache associativity\n"); + + sizes[i] = _sizes[i] / associativity; + offsets[i] = i==0 ? 0 : offsets[i-1] + sizes[i-1]; + } + + if (associativity > numEntries) + fatal("WalkCache associativity cannot be higher than " + "its number of entries\n"); + + unsigned num_sets = numEntries / associativity; + + if (num_sets*associativity != numEntries) + fatal("Number of WalkCache entries must be divisible " + "by its associativity\n"); + + Entry e; + e.valid = false; + + Set set(associativity, e); + sets.resize(num_sets, set); +} + +const WalkCache::Entry* +WalkCache::lookup(Addr va, Addr vaMask, + uint16_t asid, uint16_t vmid, + unsigned stage, unsigned level, + bool updStats) +{ + const Entry *result = NULL; + + Set &set = sets[pickSetIdx(va, vaMask, stage, level)]; + + for (size_t i = 0; i < set.size(); i++) { + const Entry &e = set[i]; + + if (e.valid && (e.va & e.vaMask) == (va & e.vaMask) && + e.asid==asid && e.vmid==vmid && e.stage==stage && e.level==level) + { + if (result != NULL) + panic("WalkCache: duplicate entry found!\n"); + + result = &e; + break; + } + } + + if (updStats) { + if (result) + result->lastUsed = useStamp++; + + totalLookups++; + if (result == NULL) + totalMisses++; + + lookupsByStageLevel[stage-1][level]++; + totalLookupsByStageLevel[stage-1][level]++; + if (result == NULL) { + missesByStageLevel[stage-1][level]++; + totalMissesByStageLevel[stage-1][level]++; + } + } + + return result; +} + +void +WalkCache::store(const Entry &incoming) +{ + if (!incoming.valid) + panic("Tried to store an invalid entry\n"); + + assert(incoming.stage==1 || incoming.stage==2); + assert(incoming.level<=WALK_CACHE_LEVELS); + + incoming.lastUsed = 0; + + const Entry *existing = lookup(incoming.va, incoming.vaMask, + incoming.asid, incoming.vmid, + incoming.stage, incoming.level, false); + + if (existing) { + *const_cast (existing) = incoming; + } else { + Set &set = sets[pickSetIdx(incoming.va, incoming.vaMask, + incoming.stage, incoming.level)]; + set[pickEntryIdxToReplace(set, incoming.stage, incoming.level)] = + incoming; + } + + totalUpdates++; + updatesByStageLevel[incoming.stage-1][incoming.level]++; + totalUpdatesByStageLevel[incoming.stage-1][incoming.level]++; +} + +void +WalkCache::invalidateVA(Addr va, uint16_t asid, uint16_t vmid) +{ + panic("%s unimplemented\n", __func__); +} + +void +WalkCache::invalidateVAA(Addr va, uint16_t vmid) +{ + panic("%s unimplemented\n", __func__); +} + +void +WalkCache::invalidateASID(uint16_t asid, uint16_t vmid) +{ + panic("%s unimplemented\n", __func__); +} + +void +WalkCache::invalidateVMID(uint16_t vmid) +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) { + Entry &e = set[i]; + + if (e.vmid == vmid) + e.valid = false; + } + } +} + +void +WalkCache::invalidateAll() +{ + for (size_t s = 0; s < sets.size(); s++) { + Set &set = sets[s]; + + for (size_t i = 0; i < set.size(); i++) + set[i].valid = false; + } +} + +size_t +WalkCache::pickSetIdx(Addr va, Addr vaMask, + unsigned stage, unsigned level) const +{ + (void) stage; + + int size, offset; + + switch (stage) { + case 1: + assert (level<=3); + size = sizes[0*WALK_CACHE_LEVELS + level]; + offset = offsets[0*WALK_CACHE_LEVELS + level]; + break; + + case 2: + assert (level<=3); + size = sizes[1*WALK_CACHE_LEVELS + level]; + offset = offsets[1*WALK_CACHE_LEVELS + level]; + break; + + default: + panic("bad stage"); + } + + return ((va >> findLsbSet(vaMask)) % size) + offset; +} + +size_t +WalkCache::pickEntryIdxToReplace(const Set &set, + unsigned stage, unsigned level) +{ + size_t lru_idx = 0; + uint32_t lru_tick = UINT32_MAX; + + for (size_t i = 0; i < set.size(); i++) { + if (!set[i].valid) { + insertions++; + insertionsByStageLevel[stage-1][level]++; + return i; + } + + if (set[i].lastUsed < lru_tick) { + lru_idx = i; + lru_tick = set[i].lastUsed; + } + } + + switch (replacementPolicy) { + case SMMU_CACHE_REPL_ROUND_ROBIN: + return nextToReplace = ((nextToReplace+1) % associativity); + + case SMMU_CACHE_REPL_RANDOM: + return random.random(0, associativity-1); + + case SMMU_CACHE_REPL_LRU: + return lru_idx; + + default: + panic("Unknown replacement policy %d\n", replacementPolicy); + } + +} + +void +WalkCache::regStats(const std::string &name) +{ + using namespace Stats; + + SMMUv3BaseCache::regStats(name); + + for (int s = 0; s < 2; s++) { + for (int l = 0; l < WALK_CACHE_LEVELS; l++) { + averageLookupsByStageLevel[s][l] + .name(csprintf("%s.averageLookupsS%dL%d", name, s+1, l)) + .desc("Average number lookups per second") + .flags(pdf); + + totalLookupsByStageLevel[s][l] + .name(csprintf("%s.totalLookupsS%dL%d", name, s+1, l)) + .desc("Total number of lookups") + .flags(pdf); + + averageLookupsByStageLevel[s][l] = + totalLookupsByStageLevel[s][l] / simSeconds; + + + averageMissesByStageLevel[s][l] + .name(csprintf("%s.averageMissesS%dL%d", name, s+1, l)) + .desc("Average number misses per second") + .flags(pdf); + + totalMissesByStageLevel[s][l] + .name(csprintf("%s.totalMissesS%dL%d", name, s+1, l)) + .desc("Total number of misses") + .flags(pdf); + + averageMissesByStageLevel[s][l] = + totalMissesByStageLevel[s][l] / simSeconds; + + + averageUpdatesByStageLevel[s][l] + .name(csprintf("%s.averageUpdatesS%dL%d", name, s+1, l)) + .desc("Average number updates per second") + .flags(pdf); + + totalUpdatesByStageLevel[s][l] + .name(csprintf("%s.totalUpdatesS%dL%d", name, s+1, l)) + .desc("Total number of updates") + .flags(pdf); + + averageUpdatesByStageLevel[s][l] = + totalUpdatesByStageLevel[s][l] / simSeconds; + + + averageHitRateByStageLevel[s][l] + .name(csprintf("%s.averageHitRateS%dL%d", name, s+1, l)) + .desc("Average hit rate") + .flags(pdf); + + averageHitRateByStageLevel[s][l] = + (totalLookupsByStageLevel[s][l] - + totalMissesByStageLevel[s][l]) + / totalLookupsByStageLevel[s][l]; + + insertionsByStageLevel[s][l] + .name(csprintf("%s.insertionsS%dL%d", name, s+1, l)) + .desc("Number of insertions (not replacements)") + .flags(pdf); + } + } +} diff --git a/src/dev/arm/smmu_v3_caches.hh b/src/dev/arm/smmu_v3_caches.hh new file mode 100644 index 000000000..ce5bb45a9 --- /dev/null +++ b/src/dev/arm/smmu_v3_caches.hh @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2014, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_CACHES_HH__ +#define __DEV_ARM_SMMU_V3_CACHES_HH__ + +#include + +#include +#include +#include +#include + +#include "base/random.hh" +#include "base/statistics.hh" +#include "base/types.hh" + +#define WALK_CACHE_LEVELS 4 + +enum { + SMMU_CACHE_REPL_ROUND_ROBIN, + SMMU_CACHE_REPL_RANDOM, + SMMU_CACHE_REPL_LRU, +}; + +class SMMUv3BaseCache +{ + protected: + int replacementPolicy; + size_t nextToReplace; + Random random; + uint32_t useStamp; + + Stats::Formula averageLookups; + Stats::Scalar totalLookups; + + Stats::Formula averageMisses; + Stats::Scalar totalMisses; + + Stats::Formula averageUpdates; + Stats::Scalar totalUpdates; + + Stats::Formula averageHitRate; + + Stats::Scalar insertions; + + static int decodePolicyName(const std::string &policy_name); + + public: + SMMUv3BaseCache(const std::string &policy_name, uint32_t seed); + virtual ~SMMUv3BaseCache() {} + + virtual void regStats(const std::string &name); +}; + +class SMMUTLB : public SMMUv3BaseCache +{ + public: + enum AllocPolicy { + ALLOC_ANY_WAY, + ALLOC_ANY_BUT_LAST_WAY, + ALLOC_LAST_WAY, + }; + + struct Entry + { + bool valid; + bool prefetched; + mutable uint32_t lastUsed; + + // TAGS + uint32_t sid; + uint32_t ssid; + Addr va; + Addr vaMask; + + // EXTRA TAGS + uint16_t asid; + uint16_t vmid; + + // OUTPUTS + Addr pa; + uint8_t permissions; + }; + + SMMUTLB(unsigned numEntries, unsigned _associativity, + const std::string &policy); + SMMUTLB(const SMMUTLB& tlb) = delete; + virtual ~SMMUTLB() {} + + const Entry *lookup(uint32_t sid, uint32_t ssid, Addr va, + bool updStats=true); + const Entry *lookupAnyVA(uint32_t sid, uint32_t ssid, + bool updStats=true); + void store(const Entry &incoming, AllocPolicy alloc); + + void invalidateVA(Addr va, uint16_t asid, uint16_t vmid); + void invalidateVAA(Addr va, uint16_t vmid); + void invalidateASID(uint16_t asid, uint16_t vmid); + void invalidateVMID(uint16_t vmid); + void invalidateAll(); + + private: + typedef std::vector Set; + std::vector sets; + + size_t associativity; + + size_t pickSetIdx(Addr va) const; + size_t pickEntryIdxToReplace(const Set &set, AllocPolicy alloc); +}; + +class ARMArchTLB : public SMMUv3BaseCache +{ + public: + struct Entry + { + bool valid; + mutable uint32_t lastUsed; + + // TAGS + Addr va; + Addr vaMask; + uint16_t asid; + uint16_t vmid; + + // OUTPUTS + Addr pa; + uint8_t permissions; + }; + + ARMArchTLB(unsigned numEntries, unsigned _associativity, + const std::string &policy); + virtual ~ARMArchTLB() {} + + const Entry *lookup(Addr va, uint16_t asid, uint16_t vmid, + bool updStats=true); + + void store(const Entry &incoming); + + void invalidateVA(Addr va, uint16_t asid, uint16_t vmid); + void invalidateVAA(Addr va, uint16_t vmid); + void invalidateASID(uint16_t asid, uint16_t vmid); + void invalidateVMID(uint16_t vmid); + void invalidateAll(); + + private: + typedef std::vector Set; + std::vector sets; + + size_t associativity; + + size_t pickSetIdx(Addr va, uint16_t asid, uint16_t vmid) const; + size_t pickEntryIdxToReplace(const Set &set); +}; + +class IPACache : public SMMUv3BaseCache +{ + public: + struct Entry + { + bool valid; + mutable uint32_t lastUsed; + + // TAGS + Addr ipa; + Addr ipaMask; + uint16_t vmid; + + // OUTPUTS + Addr pa; + uint8_t permissions; + }; + + IPACache(unsigned numEntries, unsigned _associativity, + const std::string &policy); + virtual ~IPACache() {} + + const Entry *lookup(Addr ipa, uint16_t vmid, bool updStats=true); + void store(const Entry &incoming); + + void invalidateIPA(Addr ipa, uint16_t vmid); + void invalidateIPAA(Addr ipa); + void invalidateVMID(uint16_t vmid); + void invalidateAll(); + + private: + typedef std::vector Set; + std::vector sets; + + size_t associativity; + + size_t pickSetIdx(Addr ipa, uint16_t vmid) const; + size_t pickEntryIdxToReplace(const Set &set); +}; + +class ConfigCache : public SMMUv3BaseCache +{ + public: + struct Entry + { + bool valid; + mutable uint32_t lastUsed; + + // TAGS + uint32_t sid; + uint32_t ssid; + + // OUTPUTS + bool stage1_en; + bool stage2_en; + Addr ttb0; + Addr ttb1; + Addr httb; + uint16_t asid; + uint16_t vmid; + uint8_t stage1_tg; + uint8_t stage2_tg; + }; + + ConfigCache(unsigned numEntries, unsigned _associativity, + const std::string &policy); + virtual ~ConfigCache() {} + + const Entry *lookup(uint32_t sid, uint32_t ssid, bool updStats=true); + void store(const Entry &incoming); + + void invalidateSSID(uint32_t sid, uint32_t ssid); + void invalidateSID(uint32_t sid); + void invalidateAll(); + + private: + typedef std::vector Set; + std::vector sets; + + size_t associativity; + + size_t pickSetIdx(uint32_t sid, uint32_t ssid) const; + size_t pickEntryIdxToReplace(const Set &set); +}; + +class WalkCache : public SMMUv3BaseCache +{ + public: + struct Entry + { + bool valid; + mutable uint32_t lastUsed; + + // TAGS + Addr va; + Addr vaMask; + uint16_t asid; + uint16_t vmid; + unsigned stage; + unsigned level; + + // OUTPUTS + bool leaf; + Addr pa; + uint8_t permissions; + }; + + WalkCache(const std::array &_sizes, + unsigned _associativity, const std::string &policy); + virtual ~WalkCache() {} + + const Entry *lookup(Addr va, Addr vaMask, uint16_t asid, uint16_t vmid, + unsigned stage, unsigned level, bool updStats=true); + void store(const Entry &incoming); + + void invalidateVA(Addr va, uint16_t asid, uint16_t vmid); + void invalidateVAA(Addr va, uint16_t vmid); + void invalidateASID(uint16_t asid, uint16_t vmid); + void invalidateVMID(uint16_t vmid); + void invalidateAll(); + + void regStats(const std::string &name) override; + + protected: + unsigned int lookupsByStageLevel[2][WALK_CACHE_LEVELS]; + Stats::Formula averageLookupsByStageLevel[2][WALK_CACHE_LEVELS]; + Stats::Scalar totalLookupsByStageLevel[2][WALK_CACHE_LEVELS]; + + unsigned int missesByStageLevel[2][WALK_CACHE_LEVELS]; + Stats::Formula averageMissesByStageLevel[2][WALK_CACHE_LEVELS]; + Stats::Scalar totalMissesByStageLevel[2][WALK_CACHE_LEVELS]; + + unsigned int updatesByStageLevel[2][WALK_CACHE_LEVELS]; + Stats::Formula averageUpdatesByStageLevel[2][WALK_CACHE_LEVELS]; + Stats::Scalar totalUpdatesByStageLevel[2][WALK_CACHE_LEVELS]; + + Stats::Formula averageHitRateByStageLevel[2][WALK_CACHE_LEVELS]; + + Stats::Scalar insertionsByStageLevel[2][WALK_CACHE_LEVELS]; + + private: + typedef std::vector Set; + std::vector sets; + + size_t associativity; + std::array sizes; + std::array offsets; + + size_t pickSetIdx(Addr va, Addr vaMask, + unsigned stage, unsigned level) const; + + size_t pickEntryIdxToReplace(const Set &set, + unsigned stage, unsigned level); +}; + +#endif /* __DEV_ARM_SMMU_V3_CACHES_HH__ */ diff --git a/src/dev/arm/smmu_v3_cmdexec.cc b/src/dev/arm/smmu_v3_cmdexec.cc new file mode 100644 index 000000000..8660846fd --- /dev/null +++ b/src/dev/arm/smmu_v3_cmdexec.cc @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#include "dev/arm/smmu_v3_cmdexec.hh" + +#include "base/bitfield.hh" +#include "dev/arm/smmu_v3.hh" + +void +SMMUCommandExecProcess::main(Yield &yield) +{ + SMMUAction a; + a.type = ACTION_INITIAL_NOP; + a.pkt = NULL; + a.ifc = nullptr; + a.delay = 0; + yield(a); + + while (true) { + busy = true; + + while (true) { + int sizeMask = + mask(smmu.regs.cmdq_base & Q_BASE_SIZE_MASK) & Q_CONS_PROD_MASK; + + if ((smmu.regs.cmdq_cons & sizeMask) == + (smmu.regs.cmdq_prod & sizeMask)) + break; // command queue empty + + Addr cmdAddr = + (smmu.regs.cmdq_base & Q_BASE_ADDR_MASK) + + (smmu.regs.cmdq_cons & sizeMask) * sizeof(SMMUCommand); + + // This deliberately resets the error field in cmdq_cons! + smmu.regs.cmdq_cons = (smmu.regs.cmdq_cons + 1) & sizeMask; + + doRead(yield, cmdAddr, &cmd, sizeof(SMMUCommand)); + smmu.processCommand(cmd); + } + + busy = false; + + doSleep(yield); + } +} diff --git a/src/dev/arm/smmu_v3_cmdexec.hh b/src/dev/arm/smmu_v3_cmdexec.hh new file mode 100644 index 000000000..4ce3958e7 --- /dev/null +++ b/src/dev/arm/smmu_v3_cmdexec.hh @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_CMDEXEC_HH__ +#define __DEV_ARM_SMMU_V3_CMDEXEC_HH__ + +#include "dev/arm/smmu_v3_defs.hh" +#include "dev/arm/smmu_v3_proc.hh" + +class SMMUv3; + +class SMMUCommandExecProcess : public SMMUProcess +{ + private: + SMMUCommand cmd; + + bool busy; + + virtual void main(Yield &yield); + + public: + SMMUCommandExecProcess(const std::string &name, SMMUv3 &_smmu) : + SMMUProcess(name, _smmu), + busy(false) + { + reinit(); + } + + virtual ~SMMUCommandExecProcess() {} + + bool isBusy() const { return busy; } +}; + +#endif /* __DEV_ARM_SMMU_V3_CMDEXEC_HH__ */ diff --git a/src/dev/arm/smmu_v3_defs.hh b/src/dev/arm/smmu_v3_defs.hh new file mode 100644 index 000000000..f74f8190a --- /dev/null +++ b/src/dev/arm/smmu_v3_defs.hh @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_DEFS_HH__ +#define __DEV_ARM_SMMU_V3_DEFS_HH__ + +#include + +#include "base/bitunion.hh" + +enum { + SMMU_SECURE_SZ = 0x184, // Secure regs are within page0 + SMMU_PAGE_ZERO_SZ = 0x10000, + SMMU_PAGE_ONE_SZ = 0x10000, + SMMU_REG_SIZE = SMMU_PAGE_ONE_SZ + SMMU_PAGE_ZERO_SZ +}; + +enum { + STE_CONFIG_ABORT = 0x0, + STE_CONFIG_BYPASS = 0x4, + STE_CONFIG_STAGE1_ONLY = 0x5, + STE_CONFIG_STAGE2_ONLY = 0x6, + STE_CONFIG_STAGE1_AND_2 = 0x7, +}; + +enum { + STAGE1_CFG_1L = 0x0, + STAGE1_CFG_2L_4K = 0x1, + STAGE1_CFG_2L_64K = 0x2, +}; + +enum { + ST_CFG_SPLIT_SHIFT = 6, + ST_CD_ADDR_SHIFT = 5, + CD_TTB_SHIFT = 4, + STE_S2TTB_SHIFT = 4, +}; + +enum { + TRANS_GRANULE_4K = 0x0, + TRANS_GRANULE_64K = 0x1, + TRANS_GRANULE_16K = 0x2, + TRANS_GRANULE_INVALID = 0x3, +}; + +enum { + ST_BASE_ADDR_MASK = 0x0000ffffffffffe0ULL, + ST_CFG_SIZE_MASK = 0x000000000000003fULL, + ST_CFG_SPLIT_MASK = 0x00000000000007c0ULL, + ST_CFG_FMT_MASK = 0x0000000000030000ULL, + ST_CFG_FMT_LINEAR = 0x0000000000000000ULL, + ST_CFG_FMT_2LEVEL = 0x0000000000010000ULL, + ST_L2_SPAN_MASK = 0x000000000000001fULL, + ST_L2_ADDR_MASK = 0x0000ffffffffffe0ULL, + + VMT_BASE_ADDR_MASK = 0x0000ffffffffffe0ULL, + VMT_BASE_SIZE_MASK = 0x000000000000001fULL, + + Q_CONS_PROD_MASK = 0x00000000000fffffULL, + Q_BASE_ADDR_MASK = 0x0000ffffffffffe0ULL, + Q_BASE_SIZE_MASK = 0x000000000000001fULL, + + E_BASE_ENABLE_MASK = 0x8000000000000000ULL, + E_BASE_ADDR_MASK = 0x0000fffffffffffcULL, +}; + +union SMMURegs +{ + uint8_t data[SMMU_REG_SIZE]; + + struct + { + uint32_t idr0; // 0x0000 + uint32_t idr1; // 0x0004 + uint32_t idr2; // 0x0008 + uint32_t idr3; // 0x000c + uint32_t idr4; // 0x0010 + uint32_t idr5; // 0x0014 + uint32_t iidr; // 0x0018 + uint32_t aidr; // 0x001c + uint32_t cr0; // 0x0020 + uint32_t cr0ack; // 0x0024 + uint32_t cr1; // 0x0028 + uint32_t cr2; // 0x002c + uint32_t _pad1; // 0x0030 + uint32_t _pad2; // 0x0034 + uint32_t _pad3; // 0x0038 + uint32_t _pad4; // 0x003c + uint32_t statusr; // 0x0040 + uint32_t gbpa; // 0x0044 + uint32_t agbpa; // 0x0048 + uint32_t _pad5; // 0x004c + uint32_t irq_ctrl; // 0x0050 + uint32_t irq_ctrlack; // 0x0054 + uint32_t _pad6; // 0x0058 + uint32_t _pad7; // 0x005c + + uint32_t gerror; // 0x0060 + uint32_t gerrorn; // 0x0064 + uint64_t gerror_irq_cfg0; // 0x0068, 64 bit + uint32_t gerror_irq_cfg1; // 0x0070 + uint32_t gerror_irq_cfg2; // 0x0074 + uint32_t _pad_1; // 0x0078 + uint32_t _pad_2; // 0x007c + + uint64_t strtab_base; // 0x0080, 64 bit + uint32_t strtab_base_cfg; // 0x0088 + + uint64_t cmdq_base; // 0x0090, 64 bit + uint32_t cmdq_prod; // 0x0098 + uint32_t cmdq_cons; // 0x009c + uint64_t eventq_base; // 0x00a0, 64 bit + uint32_t _pad8; // 0x00a8 + uint32_t _pad9; // 0x00ac + uint64_t eventq_irq_cfg0; // 0x00b0, 64 bit + uint32_t eventq_irq_cfg1; // 0x00b8 + uint32_t eventq_irq_cfg2; // 0x00bc + uint64_t priq_base; // 0x00c0, 64 bit + uint32_t _pad10; // 0x00c8 + uint32_t _pad11; // 0x00cc + + uint64_t priq_irq_cfg0; // 0x00d0 + uint32_t priq_irq_cfg1; // 0x00d8 + uint32_t priq_irq_cfg2; // 0x00dc + + uint32_t _pad12[8]; // 0x00e0 - 0x0100 + uint32_t gatos_ctrl; // 0x0100 + uint32_t _pad13; // 0x0104 + uint64_t gatos_sid; // 0x0108 + uint64_t gatos_addr; // 0x0110 + uint64_t gatos_par; // 0x0118 + uint32_t _pad14[24]; // 0x0120 + uint32_t vatos_sel; // 0x0180 + + uint32_t _pad15[8095]; // 0x184 - 0x7ffc + + uint8_t _secure_regs[SMMU_SECURE_SZ]; // 0x8000 - 0x8180 + + uint32_t _pad16[8095]; // 0x8184 - 0x10000 + + // Page 1 + uint32_t _pad17[42]; // 0x10000 + uint32_t eventq_prod; // 0x100A8 + uint32_t eventq_cons; // 0x100AC + + uint32_t _pad18[6]; // 0x100B0 + uint32_t priq_prod; // 0x100C8 + uint32_t priq_cons; // 0x100CC + }; +}; + +struct StreamTableEntry +{ + BitUnion64(DWORD0) + Bitfield<0> valid; + Bitfield<3, 1> config; + Bitfield<5, 4> s1fmt; + Bitfield<51, 6> s1ctxptr; + Bitfield<63, 59> s1cdmax; + EndBitUnion(DWORD0) + DWORD0 dw0; + + BitUnion64(DWORD1) + Bitfield<1, 0> s1dss; + Bitfield<3, 2> s1cir; + Bitfield<5, 4> s1cor; + Bitfield<7, 6> s1csh; + Bitfield<8> s2hwu59; + Bitfield<9> s2hwu60; + Bitfield<10> s2hwu61; + Bitfield<11> s2hwu62; + Bitfield<12> dre; + Bitfield<16, 13> cont; + Bitfield<17> dcp; + Bitfield<18> ppar; + Bitfield<19> mev; + Bitfield<27> s1stalld; + Bitfield<29, 28> eats; + Bitfield<31, 30> strw; + Bitfield<35, 32> memattr; + Bitfield<36> mtcfg; + Bitfield<40, 37> alloccfg; + Bitfield<45, 44> shcfg; + Bitfield<47, 46> nscfg; + Bitfield<49, 48> privcfg; + Bitfield<51, 50> instcfg; + EndBitUnion(DWORD1) + DWORD1 dw1; + + BitUnion64(DWORD2) + Bitfield<15, 0> s2vmid; + Bitfield<37, 32> s2t0sz; + Bitfield<39, 38> s2sl0; + Bitfield<41, 40> s2ir0; + Bitfield<43, 42> s2or0; + Bitfield<45, 44> s2sh0; + Bitfield<47, 46> s2tg; + Bitfield<50, 48> s2ps; + Bitfield<51> s2aa64; + Bitfield<52> s2endi; + Bitfield<53> s2affd; + Bitfield<54> s2ptw; + Bitfield<55> s2hd; + Bitfield<56> s2ha; + Bitfield<57> s2s; + Bitfield<58> s2r; + EndBitUnion(DWORD2) + DWORD2 dw2; + + BitUnion64(DWORD3) + Bitfield<51, 4> s2ttb; + EndBitUnion(DWORD3) + DWORD3 dw3; + + uint64_t _pad[4]; +}; + +struct ContextDescriptor +{ + BitUnion64(DWORD0) + Bitfield<5, 0> t0sz; + Bitfield<7, 6> tg0; + Bitfield<9, 8> ir0; + Bitfield<11, 10> or0; + Bitfield<13, 12> sh0; + Bitfield<14> epd0; + Bitfield<15> endi; + Bitfield<21, 16> t1sz; + Bitfield<23, 22> tg1; + Bitfield<25, 24> ir1; + Bitfield<27, 26> or1; + Bitfield<29, 28> sh1; + Bitfield<30> epd1; + Bitfield<31> valid; + Bitfield<34, 32> ips; + Bitfield<35> affd; + Bitfield<36> wxn; + Bitfield<37> uwxn; + Bitfield<39, 38> tbi; + Bitfield<40> pan; + Bitfield<41> aa64; + Bitfield<42> hd; + Bitfield<43> ha; + Bitfield<44> s; + Bitfield<45> r; + Bitfield<46> a; + Bitfield<47> aset; + Bitfield<63, 48> asid; + EndBitUnion(DWORD0) + DWORD0 dw0; + + BitUnion64(DWORD1) + Bitfield<0> nscfg0; + Bitfield<1> had0; + Bitfield<51, 4> ttb0; + Bitfield<60> hwu0g59; + Bitfield<61> hwu0g60; + Bitfield<62> hwu0g61; + Bitfield<63> hwu0g62; + EndBitUnion(DWORD1) + DWORD1 dw1; + + BitUnion64(DWORD2) + Bitfield<0> nscfg1; + Bitfield<1> had1; + Bitfield<51, 4> ttb1; + Bitfield<60> hwu1g59; + Bitfield<61> hwu1g60; + Bitfield<62> hwu1g61; + Bitfield<63> hwu1g62; + EndBitUnion(DWORD2) + DWORD2 dw2; + + uint64_t mair; + uint64_t amair; + uint64_t _pad[3]; +}; + +enum SMMUCommandType { + CMD_PRF_CONFIG = 0x1000, + CMD_PRF_ADDR = 0x1001, + CMD_INV_STE = 0x1100, + CMD_INV_CD = 0x1101, + CMD_INV_CD_ALL = 0x1102, + CMD_INV_ALL = 0x1104, + CMD_TLBI_ALL = 0x1110, + CMD_TLBI_ASID = 0x1111, + CMD_TLBI_VAAL = 0x1112, + CMD_TLBI_VAA = 0x1113, + CMD_TLBI_VAL = 0x1114, + CMD_TLBI_VA = 0x1115, + CMD_TLBI_VM_IPAL = 0x1120, + CMD_TLBI_VM_IPA = 0x1121, + CMD_TLBI_VM_S12 = 0x1122, + CMD_RESUME_S = 0x1200, +}; + +struct SMMUCommand +{ + uint32_t type; + uint32_t data[3]; +}; + +enum SMMUEventTypes { + EVT_FAULT = 0x0001, +}; + +enum SMMUEventFlags { + EVF_WRITE = 0x0001, +}; + +struct SMMUEvent +{ + uint16_t type; + uint16_t stag; + uint32_t flags; + uint32_t streamId; + uint32_t substreamId; + uint64_t va; + uint64_t ipa; +}; + +enum { + SMMU_MAX_TRANS_ID = 64 +}; + +#endif /* __DEV_ARM_SMMU_V3_DEFS_HH__ */ diff --git a/src/dev/arm/smmu_v3_events.cc b/src/dev/arm/smmu_v3_events.cc new file mode 100644 index 000000000..e548fb0b0 --- /dev/null +++ b/src/dev/arm/smmu_v3_events.cc @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2014, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#include "dev/arm/smmu_v3_events.hh" + +#include "dev/arm/smmu_v3_slaveifc.hh" + +void +SMMUDeviceRetryEvent::process() +{ + smmuIfc.sendDeviceRetry(); +} + +const std::string +SMMUDeviceRetryEvent::name() const +{ + return smmuIfc.name() + ".device_retry_event"; +} diff --git a/src/dev/arm/smmu_v3_events.hh b/src/dev/arm/smmu_v3_events.hh new file mode 100644 index 000000000..21dbca862 --- /dev/null +++ b/src/dev/arm/smmu_v3_events.hh @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2014, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_EVENTS_HH__ +#define __DEV_ARM_SMMU_V3_EVENTS_HH__ + +#include +#include + +class SMMUv3SlaveInterface; + +class SMMUDeviceRetryEvent : public Event +{ + private: + SMMUv3SlaveInterface &smmuIfc; + + public: + SMMUDeviceRetryEvent(SMMUv3SlaveInterface &ifc) + : smmuIfc(ifc) + {} + + void process(); + + const std::string name() const; + + const char *description() const + { return "SlaveRetryEvent"; } +}; + +#endif /* __DEV_ARM_SMMU_V3_EVENTS_HH__ */ diff --git a/src/dev/arm/smmu_v3_ports.cc b/src/dev/arm/smmu_v3_ports.cc new file mode 100644 index 000000000..178592587 --- /dev/null +++ b/src/dev/arm/smmu_v3_ports.cc @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#include "dev/arm/smmu_v3_ports.hh" + +#include "base/logging.hh" +#include "dev/arm/smmu_v3.hh" +#include "dev/arm/smmu_v3_slaveifc.hh" + +SMMUMasterPort::SMMUMasterPort(const std::string &_name, SMMUv3 &_smmu) : + MasterPort(_name, &_smmu), + smmu(_smmu) +{} + +bool +SMMUMasterPort::recvTimingResp(PacketPtr pkt) +{ + return smmu.masterRecvTimingResp(pkt); +} + +void +SMMUMasterPort::recvReqRetry() +{ + return smmu.masterRecvReqRetry(); +} + +SMMUMasterTableWalkPort::SMMUMasterTableWalkPort(const std::string &_name, + SMMUv3 &_smmu) : + MasterPort(_name, &_smmu), + smmu(_smmu) +{} + +bool +SMMUMasterTableWalkPort::recvTimingResp(PacketPtr pkt) +{ + return smmu.masterTableWalkRecvTimingResp(pkt); +} + +void +SMMUMasterTableWalkPort::recvReqRetry() +{ + return smmu.masterTableWalkRecvReqRetry(); +} + +SMMUSlavePort::SMMUSlavePort(const std::string &_name, + SMMUv3SlaveInterface &_ifc, + PortID _id) +: + QueuedSlavePort(_name, &_ifc, respQueue, _id), + ifc(_ifc), + respQueue(_ifc, *this) +{} + +void +SMMUSlavePort::recvFunctional(PacketPtr pkt) +{ + if (!respQueue.trySatisfyFunctional(pkt)) + recvAtomic(pkt); +} + +Tick +SMMUSlavePort::recvAtomic(PacketPtr pkt) +{ + return ifc.recvAtomic(pkt); +} + +bool +SMMUSlavePort::recvTimingReq(PacketPtr pkt) +{ + return ifc.recvTimingReq(pkt); +} + +SMMUControlPort::SMMUControlPort(const std::string &_name, + SMMUv3 &_smmu, AddrRange _addrRange) +: + SimpleTimingPort(_name, &_smmu), + smmu(_smmu), + addrRange(_addrRange) +{} + +Tick +SMMUControlPort::recvAtomic(PacketPtr pkt) +{ + Addr addr = pkt->getAddr(); + unsigned size = pkt->getSize(); + + if (!addrRange.contains(addr) || !addrRange.contains(addr+size)) + panic("SMMU: invalid address on control port %x, packet size %d", + addr, size); + + // @todo: We need to pay for this and not just zero it out + pkt->headerDelay = pkt->payloadDelay = 0; + + return pkt->isRead() ? smmu.readControl(pkt) : smmu.writeControl(pkt); +} + +AddrRangeList +SMMUControlPort::getAddrRanges() const +{ + AddrRangeList list; + list.push_back(addrRange); + return list; +} + +SMMUATSMasterPort::SMMUATSMasterPort(const std::string &_name, + SMMUv3SlaveInterface &_ifc) : + QueuedMasterPort(_name, &_ifc, reqQueue, snoopRespQueue), + ifc(_ifc), + reqQueue(_ifc, *this), + snoopRespQueue(_ifc, *this) +{} + +bool +SMMUATSMasterPort::recvTimingResp(PacketPtr pkt) +{ + return ifc.atsMasterRecvTimingResp(pkt); +} + +SMMUATSSlavePort::SMMUATSSlavePort(const std::string &_name, + SMMUv3SlaveInterface &_ifc) : + QueuedSlavePort(_name, &_ifc, respQueue), + ifc(_ifc), + respQueue(_ifc, *this) +{} + +void +SMMUATSSlavePort::recvFunctional(PacketPtr pkt) +{ + panic("Functional access on ATS port!"); +} + +Tick +SMMUATSSlavePort::recvAtomic(PacketPtr pkt) +{ + return ifc.atsSlaveRecvAtomic(pkt); +} + +bool +SMMUATSSlavePort::recvTimingReq(PacketPtr pkt) +{ + return ifc.atsSlaveRecvTimingReq(pkt); +} diff --git a/src/dev/arm/smmu_v3_ports.hh b/src/dev/arm/smmu_v3_ports.hh new file mode 100644 index 000000000..bdd10e568 --- /dev/null +++ b/src/dev/arm/smmu_v3_ports.hh @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_PORTS_HH__ +#define __DEV_ARM_SMMU_V3_PORTS_HH__ + +#include "mem/qport.hh" +#include "mem/tport.hh" + +class SMMUv3; +class SMMUv3SlaveInterface; + +class SMMUMasterPort : public MasterPort +{ + protected: + SMMUv3 &smmu; + + virtual bool recvTimingResp(PacketPtr pkt); + virtual void recvReqRetry(); + + public: + SMMUMasterPort(const std::string &_name, SMMUv3 &_smmu); + virtual ~SMMUMasterPort() {} +}; + +// Separate master port to send MMU initiated requests on +class SMMUMasterTableWalkPort : public MasterPort +{ + protected: + SMMUv3 &smmu; + + virtual bool recvTimingResp(PacketPtr pkt); + virtual void recvReqRetry(); + + public: + SMMUMasterTableWalkPort(const std::string &_name, SMMUv3 &_smmu); + virtual ~SMMUMasterTableWalkPort() {} +}; + +class SMMUSlavePort : public QueuedSlavePort +{ + protected: + SMMUv3SlaveInterface &ifc; + RespPacketQueue respQueue; + + virtual void recvFunctional(PacketPtr pkt); + virtual Tick recvAtomic(PacketPtr pkt); + virtual bool recvTimingReq(PacketPtr pkt); + + public: + SMMUSlavePort(const std::string &_name, + SMMUv3SlaveInterface &_ifc, + PortID _id = InvalidPortID); + virtual ~SMMUSlavePort() {} + + virtual AddrRangeList getAddrRanges() const + { return AddrRangeList { AddrRange(0, UINT64_MAX) }; } +}; + +class SMMUControlPort : public SimpleTimingPort +{ + protected: + SMMUv3 &smmu; + AddrRange addrRange; + + virtual Tick recvAtomic(PacketPtr pkt); + virtual AddrRangeList getAddrRanges() const; + + public: + SMMUControlPort(const std::string &_name, SMMUv3 &_smmu, + AddrRange _addrRange); + virtual ~SMMUControlPort() {} +}; + +class SMMUATSMasterPort : public QueuedMasterPort +{ + protected: + SMMUv3SlaveInterface &ifc; + ReqPacketQueue reqQueue; + SnoopRespPacketQueue snoopRespQueue; + + virtual bool recvTimingResp(PacketPtr pkt); + + public: + SMMUATSMasterPort(const std::string &_name, SMMUv3SlaveInterface &_ifc); + virtual ~SMMUATSMasterPort() {} +}; + +class SMMUATSSlavePort : public QueuedSlavePort +{ + protected: + SMMUv3SlaveInterface &ifc; + RespPacketQueue respQueue; + + virtual void recvFunctional(PacketPtr pkt); + virtual Tick recvAtomic(PacketPtr pkt); + virtual bool recvTimingReq(PacketPtr pkt); + + virtual AddrRangeList getAddrRanges() const + { return AddrRangeList(); } + + public: + SMMUATSSlavePort(const std::string &_name, SMMUv3SlaveInterface &_ifc); + virtual ~SMMUATSSlavePort() {} +}; + +#endif /* __DEV_ARM_SMMU_V3_PORTS_HH__ */ diff --git a/src/dev/arm/smmu_v3_proc.cc b/src/dev/arm/smmu_v3_proc.cc new file mode 100644 index 000000000..71888bd8b --- /dev/null +++ b/src/dev/arm/smmu_v3_proc.cc @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#include "dev/arm/smmu_v3_proc.hh" + +#include "dev/arm/smmu_v3.hh" +#include "sim/system.hh" + +SMMUProcess::SMMUProcess(const std::string &name, SMMUv3 &_smmu) : + coroutine(NULL), + myName(name), + smmu(_smmu) +{} + +SMMUProcess::~SMMUProcess() +{ + delete coroutine; +} + +void +SMMUProcess::wakeup() +{ + smmu.runProcess(this, NULL); +} + +void +SMMUProcess::reinit() +{ + delete coroutine; + coroutine = new Coroutine( + std::bind(&SMMUProcess::main, this, std::placeholders::_1)); +} + +void +SMMUProcess::doRead(Yield &yield, Addr addr, void *ptr, size_t size) +{ + doSemaphoreDown(yield, smmu.masterPortSem); + doDelay(yield, Cycles(1)); // request - assume 1 cycle + doSemaphoreUp(smmu.masterPortSem); + + SMMUAction a; + a.type = ACTION_SEND_REQ; + + RequestPtr req = std::make_shared( + addr, size, 0, smmu.masterId); + + req->taskId(ContextSwitchTaskId::DMA); + + a.pkt = new Packet(req, MemCmd::ReadReq); + a.pkt->dataStatic(ptr); + + a.delay = 0; + + PacketPtr pkt = yield(a).get(); + + assert(pkt); + // >= because we may get the whole cache line + assert(pkt->getSize() >= size); + + delete pkt; +} + +void +SMMUProcess::doWrite(Yield &yield, Addr addr, const void *ptr, size_t size) +{ + unsigned nbeats = (size + (smmu.masterPortWidth-1)) / smmu.masterPortWidth; + + doSemaphoreDown(yield, smmu.masterPortSem); + doDelay(yield, Cycles(nbeats)); + doSemaphoreUp(smmu.masterPortSem); + + + SMMUAction a; + a.type = ACTION_SEND_REQ; + + RequestPtr req = std::make_shared( + addr, size, 0, smmu.masterId); + + req->taskId(ContextSwitchTaskId::DMA); + + a.pkt = new Packet(req, MemCmd::WriteReq); + a.pkt->dataStatic(ptr); + + PacketPtr pkt = yield(a).get(); + + delete pkt; +} + +void +SMMUProcess::doDelay(Yield &yield, Cycles cycles) +{ + if (smmu.system.isTimingMode()) + scheduleWakeup(smmu.clockEdge(cycles)); + + SMMUAction a; + a.type = ACTION_DELAY; + a.delay = cycles * smmu.clockPeriod(); + yield(a); +} + +void +SMMUProcess::doSleep(Yield &yield) +{ + SMMUAction a; + a.type = ACTION_SLEEP; + yield(a); +} + +void +SMMUProcess::doSemaphoreDown(Yield &yield, SMMUSemaphore &sem) +{ + while (sem.count == 0) { + sem.queue.push(this); + doSleep(yield); + } + + sem.count--; + return; +} + +void +SMMUProcess::doSemaphoreUp(SMMUSemaphore &sem) +{ + sem.count++; + if (!sem.queue.empty()) { + SMMUProcess *next_proc = sem.queue.front(); + sem.queue.pop(); + + // Schedule event in the current tick instead of + // calling the function directly to avoid overflowing + // the stack in this coroutine. + next_proc->scheduleWakeup(curTick()); + } +} + +void +SMMUProcess::doWaitForSignal(Yield &yield, SMMUSignal &sig) +{ + sig.waiting.push_back(this); + doSleep(yield); +} + +void +SMMUProcess::doBroadcastSignal(SMMUSignal &sig) +{ + if (!sig.waiting.empty()) { + for (auto it : sig.waiting) { + // Schedule event in the current tick instead of + // calling the function directly to avoid overflowing + // the stack in this coroutine. + it->scheduleWakeup(curTick()); + } + + sig.waiting.clear(); + } +} + +void +SMMUProcess::scheduleWakeup(Tick when) +{ + auto *ep = new EventWrapper< + SMMUProcess, &SMMUProcess::wakeup> (this, true); + + smmu.schedule(ep, when); +} + +SMMUAction +SMMUProcess::run(PacketPtr pkt) +{ + assert(coroutine != NULL); + assert(*coroutine); + return (*coroutine)(pkt).get(); +} diff --git a/src/dev/arm/smmu_v3_proc.hh b/src/dev/arm/smmu_v3_proc.hh new file mode 100644 index 000000000..9444c01b0 --- /dev/null +++ b/src/dev/arm/smmu_v3_proc.hh @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_PROC_HH__ +#define __DEV_ARM_SMMU_V3_PROC_HH__ + +#include +#include +#include + +#include "base/coroutine.hh" +#include "base/types.hh" +#include "mem/packet.hh" + +class SMMUv3SlaveInterface; + +/* + * The meaning of these becomes apparent when you + * look at runProcessAtomic()/runProcessTiming(). + */ +enum SMMUActionType { + ACTION_INITIAL_NOP, + ACTION_SEND_REQ, + ACTION_SEND_REQ_FINAL, + ACTION_SEND_RESP, + ACTION_SEND_RESP_ATS, + ACTION_DELAY, + ACTION_SLEEP, + ACTION_TERMINATE, +}; + +struct SMMUAction +{ + SMMUActionType type; + PacketPtr pkt; + SMMUv3SlaveInterface *ifc; + Tick delay; +}; + +class SMMUv3; +class SMMUProcess; + +struct SMMUSemaphore +{ + explicit SMMUSemaphore(unsigned _max) : + count(_max), max(_max) + {} + + unsigned count; + unsigned max; + std::queue queue; +}; + +struct SMMUSignal +{ + std::list waiting; +}; + +class SMMUProcess : public Packet::SenderState +{ + private: + typedef m5::Coroutine Coroutine; + + Coroutine *coroutine; + std::string myName; + + void wakeup(); + + protected: + typedef Coroutine::CallerType Yield; + + SMMUv3 &smmu; + + void reinit(); + + virtual void main(Yield &yield) = 0; + + void doRead(Yield &yield, Addr addr, void *ptr, size_t size); + void doWrite(Yield &yield, Addr addr, const void *ptr, size_t size); + void doDelay(Yield &yield, Cycles cycles); + void doSleep(Yield &yield); + + void doSemaphoreDown(Yield &yield, SMMUSemaphore &sem); + void doSemaphoreUp(SMMUSemaphore &sem); + + void doWaitForSignal(Yield &yield, SMMUSignal &sig); + void doBroadcastSignal(SMMUSignal &sig); + + void scheduleWakeup(Tick when); + + public: + SMMUProcess(const std::string &name, SMMUv3 &_smmu); + virtual ~SMMUProcess(); + + SMMUAction run(PacketPtr pkt); + + const std::string name() const { return myName; }; +}; + +#endif /* __DEV_ARM_SMMU_V3_PROC_HH__ */ diff --git a/src/dev/arm/smmu_v3_ptops.cc b/src/dev/arm/smmu_v3_ptops.cc new file mode 100644 index 000000000..6490b34c7 --- /dev/null +++ b/src/dev/arm/smmu_v3_ptops.cc @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#include "dev/arm/smmu_v3_ptops.hh" + +#include "base/bitfield.hh" +#include "base/logging.hh" + +bool +V7LPageTableOps::isValid(pte_t pte, unsigned level) const +{ + switch (level) { + case 1: return pte & 0x1; + case 2: return pte & 0x1; + case 3: return (pte & 0x1) && (pte & 0x2); + default: panic("bad level %d", level); + } +} + +bool +V7LPageTableOps::isLeaf(pte_t pte, unsigned level) const +{ + switch (level) { + case 1: return !(pte & 0x2); + case 2: return !(pte & 0x2); + case 3: return true; + default: panic("bad level %d", level); + } +} + +bool +V7LPageTableOps::isWritable(pte_t pte, unsigned level, bool stage2) const +{ + return stage2 ? bits(pte, 7, 6)==3 : bits(pte, 7)==0; +} + +Addr +V7LPageTableOps::nextLevelPointer(pte_t pte, unsigned level) const +{ + if (isLeaf(pte, level)) { + switch (level) { + case 1: return mbits(pte, 39, 30); + case 2: return mbits(pte, 39, 21); + case 3: return mbits(pte, 39, 12); + default: panic("bad level %d", level); + } + } else { + return mbits(pte, 39, 12); + } +} + +Addr +V7LPageTableOps::index(Addr va, unsigned level) const +{ + // In theory this should be configurable... + const int n = 12; + + switch (level) { + case 1: return bits(va, 26+n, 30) << 3; break; + case 2: return bits(va, 29, 21) << 3; break; + case 3: return bits(va, 20, 12) << 3; break; + default: panic("bad level %d", level); + } +} + +Addr +V7LPageTableOps::pageMask(pte_t pte, unsigned level) const +{ + switch (level) { + case 1: return ~mask(30); + case 2: return ~mask(21); + case 3: return bits(pte, 52) ? ~mask(16) : ~mask(12); + default: panic("bad level %d", level); + } +} + +Addr +V7LPageTableOps::walkMask(unsigned level) const +{ + switch (level) { + case 1: return mask(39, 30); + case 2: return mask(39, 21); + case 3: return mask(39, 12); + default: panic("bad level %d", level); + } +} + +unsigned +V7LPageTableOps::firstLevel() const +{ + return 1; +} + +unsigned +V7LPageTableOps::lastLevel() const +{ + return 3; +} + +bool +V8PageTableOps4k::isValid(pte_t pte, unsigned level) const +{ + switch (level) { + case 0: return pte & 0x1; + case 1: return pte & 0x1; + case 2: return pte & 0x1; + case 3: return (pte & 0x1) && (pte & 0x2); + default: panic("bad level %d", level); + } +} + +bool +V8PageTableOps4k::isLeaf(pte_t pte, unsigned level) const +{ + switch (level) { + case 0: return false; + case 1: return !(pte & 0x2); + case 2: return !(pte & 0x2); + case 3: return true; + default: panic("bad level %d", level); + } +} + +bool +V8PageTableOps4k::isWritable(pte_t pte, unsigned level, bool stage2) const +{ + return stage2 ? bits(pte, 7, 6)==3 : bits(pte, 7)==0; +} + +Addr +V8PageTableOps4k::nextLevelPointer(pte_t pte, unsigned level) const +{ + if (isLeaf(pte, level)) { + switch (level) { + // no level 0 here + case 1: return mbits(pte, 47, 30); + case 2: return mbits(pte, 47, 21); + case 3: return mbits(pte, 47, 12); + default: panic("bad level %d", level); + } + } else { + return mbits(pte, 47, 12); + } +} + +Addr +V8PageTableOps4k::index(Addr va, unsigned level) const +{ + switch (level) { + case 0: return bits(va, 47, 39) << 3; break; + case 1: return bits(va, 38, 30) << 3; break; + case 2: return bits(va, 29, 21) << 3; break; + case 3: return bits(va, 20, 12) << 3; break; + default: panic("bad level %d", level); + } +} + +Addr +V8PageTableOps4k::pageMask(pte_t pte, unsigned level) const +{ + switch (level) { + // no level 0 here + case 1: return ~mask(30); + case 2: return ~mask(21); + case 3: return bits(pte, 52) ? ~mask(16) : ~mask(12); + default: panic("bad level %d", level); + } +} + +Addr +V8PageTableOps4k::walkMask(unsigned level) const +{ + switch (level) { + case 0: return mask(47, 39); + case 1: return mask(47, 30); + case 2: return mask(47, 21); + case 3: return mask(47, 12); + default: panic("bad level %d", level); + } +} + +unsigned +V8PageTableOps4k::firstLevel() const +{ + return 0; +} + +unsigned +V8PageTableOps4k::lastLevel() const +{ + return 3; +} + +bool +V8PageTableOps64k::isValid(pte_t pte, unsigned level) const +{ + switch (level) { + case 1: return pte & 0x1; + case 2: return pte & 0x1; + case 3: return (pte & 0x1) && (pte & 0x2); + default: panic("bad level %d", level); + } +} + +bool +V8PageTableOps64k::isLeaf(pte_t pte, unsigned level) const +{ + switch (level) { + case 1: return false; + case 2: return !(pte & 0x2); + case 3: return true; + default: panic("bad level %d", level); + } +} + +bool +V8PageTableOps64k::isWritable(pte_t pte, unsigned level, bool stage2) const +{ + return stage2 ? bits(pte, 7, 6)==3 : bits(pte, 7)==0; +} + +Addr +V8PageTableOps64k::nextLevelPointer(pte_t pte, unsigned level) const +{ + if (isLeaf(pte, level)) { + switch (level) { + // no level 1 here + case 2: return mbits(pte, 47, 29); + case 3: return mbits(pte, 47, 16); + default: panic("bad level %d", level); + } + } else { + return mbits(pte, 47, 16); + } +} + +Addr +V8PageTableOps64k::index(Addr va, unsigned level) const +{ + switch (level) { + case 1: return bits(va, 47, 42) << 3; break; + case 2: return bits(va, 41, 29) << 3; break; + case 3: return bits(va, 28, 16) << 3; break; + default: panic("bad level %d", level); + } +} + +Addr +V8PageTableOps64k::pageMask(pte_t pte, unsigned level) const +{ + switch (level) { + // no level 1 here + case 2: return ~mask(29); + case 3: return bits(pte, 52) ? ~mask(21) : ~mask(16); + default: panic("bad level %d", level); + } +} + +Addr +V8PageTableOps64k::walkMask(unsigned level) const +{ + switch (level) { + case 1: return mask(47, 42); + case 2: return mask(47, 29); + case 3: return mask(47, 16); + default: panic("bad level %d", level); + } +} + +unsigned +V8PageTableOps64k::firstLevel() const +{ + return 1; +} + +unsigned +V8PageTableOps64k::lastLevel() const +{ + return 3; +} diff --git a/src/dev/arm/smmu_v3_ptops.hh b/src/dev/arm/smmu_v3_ptops.hh new file mode 100644 index 000000000..ef9ef0409 --- /dev/null +++ b/src/dev/arm/smmu_v3_ptops.hh @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_PTOPS_HH__ +#define __DEV_ARM_SMMU_V3_PTOPS_HH__ + +#include + +#include "base/types.hh" + +struct PageTableOps +{ + typedef int64_t pte_t; + + virtual bool isValid(pte_t pte, unsigned level) const = 0; + virtual bool isLeaf(pte_t pte, unsigned level) const = 0; + virtual bool isWritable(pte_t pte, unsigned level, bool stage2) const = 0; + virtual Addr nextLevelPointer(pte_t pte, unsigned level) const = 0; + virtual Addr index(Addr va, unsigned level) const = 0; + virtual Addr pageMask(pte_t pte, unsigned level) const = 0; + virtual Addr walkMask(unsigned level) const = 0; + virtual unsigned firstLevel() const = 0; + virtual unsigned lastLevel() const = 0; +}; + +struct V7LPageTableOps : public PageTableOps +{ + virtual bool isValid(pte_t pte, unsigned level) const; + virtual bool isLeaf(pte_t pte, unsigned level) const; + virtual bool isWritable(pte_t pte, unsigned level, bool stage2) const; + virtual Addr nextLevelPointer(pte_t pte, unsigned level) const; + virtual Addr index(Addr va, unsigned level) const; + virtual Addr pageMask(pte_t pte, unsigned level) const; + virtual Addr walkMask(unsigned level) const; + virtual unsigned firstLevel() const; + virtual unsigned lastLevel() const; +}; + +struct V8PageTableOps4k : public PageTableOps +{ + virtual bool isValid(pte_t pte, unsigned level) const; + virtual bool isLeaf(pte_t pte, unsigned level) const; + virtual bool isWritable(pte_t pte, unsigned level, bool stage2) const; + virtual Addr nextLevelPointer(pte_t pte, unsigned level) const; + virtual Addr index(Addr va, unsigned level) const; + virtual Addr pageMask(pte_t pte, unsigned level) const; + virtual Addr walkMask(unsigned level) const; + virtual unsigned firstLevel() const; + virtual unsigned lastLevel() const; +}; + +struct V8PageTableOps64k : public PageTableOps +{ + virtual bool isValid(pte_t pte, unsigned level) const; + virtual bool isLeaf(pte_t pte, unsigned level) const; + virtual bool isWritable(pte_t pte, unsigned level, bool stage2) const; + virtual Addr nextLevelPointer(pte_t pte, unsigned level) const; + virtual Addr index(Addr va, unsigned level) const; + virtual Addr pageMask(pte_t pte, unsigned level) const; + virtual Addr walkMask(unsigned level) const; + virtual unsigned firstLevel() const; + virtual unsigned lastLevel() const; +}; + +#endif /* __DEV_ARM_SMMU_V3_PTOPS_HH__ */ diff --git a/src/dev/arm/smmu_v3_slaveifc.cc b/src/dev/arm/smmu_v3_slaveifc.cc new file mode 100644 index 000000000..72c319d07 --- /dev/null +++ b/src/dev/arm/smmu_v3_slaveifc.cc @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + * Giacomo Travaglini + */ + +#include "dev/arm/smmu_v3_slaveifc.hh" + +#include "debug/SMMUv3.hh" +#include "dev/arm/smmu_v3.hh" +#include "dev/arm/smmu_v3_transl.hh" + +SMMUv3SlaveInterface::SMMUv3SlaveInterface( + const SMMUv3SlaveInterfaceParams *p) : + MemObject(p), + smmu(nullptr), + microTLB(new SMMUTLB(p->utlb_entries, + p->utlb_assoc, + p->utlb_policy)), + mainTLB(new SMMUTLB(p->tlb_entries, + p->tlb_assoc, + p->tlb_policy)), + microTLBEnable(p->utlb_enable), + mainTLBEnable(p->tlb_enable), + slavePortSem(1), + microTLBSem(p->utlb_slots), + mainTLBSem(p->tlb_slots), + microTLBLat(p->utlb_lat), + mainTLBLat(p->tlb_lat), + slavePort(new SMMUSlavePort(csprintf("%s.slave", name()), *this)), + atsSlavePort(name() + ".atsSlave", *this), + atsMasterPort(name() + ".atsMaster", *this), + portWidth(p->port_width), + wrBufSlotsRemaining(p->wrbuf_slots), + xlateSlotsRemaining(p->xlate_slots), + prefetchEnable(p->prefetch_enable), + prefetchReserveLastWay( + p->prefetch_reserve_last_way), + deviceNeedsRetry(false), + atsDeviceNeedsRetry(false), + sendDeviceRetryEvent(*this), + atsSendDeviceRetryEvent(this) +{} + +void +SMMUv3SlaveInterface::sendRange() +{ + if (slavePort->isConnected()) { + inform("Slave port is connected to %d\n", + slavePort->getMasterPort().name()); + + slavePort->sendRangeChange(); + } else { + fatal("Slave port is not connected.\n"); + } +} + +Port& +SMMUv3SlaveInterface::getPort(const std::string &name, PortID id) +{ + if (name == "ats_master") { + return atsMasterPort; + } else if (name == "slave") { + return *slavePort; + } else if (name == "ats_slave") { + return atsSlavePort; + } else { + return MemObject::getPort(name, id); + } +} + +void +SMMUv3SlaveInterface::schedTimingResp(PacketPtr pkt) +{ + slavePort->schedTimingResp(pkt, nextCycle()); +} + +void +SMMUv3SlaveInterface::schedAtsTimingResp(PacketPtr pkt) +{ + atsSlavePort.schedTimingResp(pkt, nextCycle()); + + if (atsDeviceNeedsRetry) { + atsDeviceNeedsRetry = false; + schedule(atsSendDeviceRetryEvent, nextCycle()); + } +} + +Tick +SMMUv3SlaveInterface::recvAtomic(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "[a] req from %s addr=%#x size=%#x\n", + slavePort->getMasterPort().name(), + pkt->getAddr(), pkt->getSize()); + + std::string proc_name = csprintf("%s.port", name()); + SMMUTranslationProcess proc(proc_name, *smmu, *this); + proc.beginTransaction(SMMUTranslRequest::fromPacket(pkt)); + + SMMUAction a = smmu->runProcessAtomic(&proc, pkt); + assert(a.type == ACTION_SEND_RESP); + + return a.delay; +} + +bool +SMMUv3SlaveInterface::recvTimingReq(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "[t] req from %s addr=%#x size=%#x\n", + slavePort->getMasterPort().name(), + pkt->getAddr(), pkt->getSize()); + + // @todo: We need to pay for this and not just zero it out + pkt->headerDelay = pkt->payloadDelay = 0; + + unsigned nbeats = + (pkt->getSize() + (portWidth-1)) / portWidth; + + if (xlateSlotsRemaining==0 || + (pkt->isWrite() && wrBufSlotsRemaining < nbeats)) + { + deviceNeedsRetry = true; + return false; + } + + xlateSlotsRemaining--; + if (pkt->isWrite()) + wrBufSlotsRemaining -= nbeats; + + std::string proc_name = csprintf("%s.port", name()); + SMMUTranslationProcess *proc = + new SMMUTranslationProcess(proc_name, *smmu, *this); + proc->beginTransaction(SMMUTranslRequest::fromPacket(pkt)); + + smmu->runProcessTiming(proc, pkt); + + return true; +} + +Tick +SMMUv3SlaveInterface::atsSlaveRecvAtomic(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "[a] ATS slave req addr=%#x size=%#x\n", + pkt->getAddr(), pkt->getSize()); + + std::string proc_name = csprintf("%s.atsport", name()); + const bool ats_request = true; + SMMUTranslationProcess proc( + proc_name, *smmu, *this); + proc.beginTransaction(SMMUTranslRequest::fromPacket(pkt, ats_request)); + + SMMUAction a = smmu->runProcessAtomic(&proc, pkt); + assert(a.type == ACTION_SEND_RESP_ATS); + + return a.delay; +} + +bool +SMMUv3SlaveInterface::atsSlaveRecvTimingReq(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "[t] ATS slave req addr=%#x size=%#x\n", + pkt->getAddr(), pkt->getSize()); + + // @todo: We need to pay for this and not just zero it out + pkt->headerDelay = pkt->payloadDelay = 0; + + if (xlateSlotsRemaining == 0) { + deviceNeedsRetry = true; + return false; + } + + xlateSlotsRemaining--; + + std::string proc_name = csprintf("%s.atsport", name()); + const bool ats_request = true; + SMMUTranslationProcess *proc = + new SMMUTranslationProcess(proc_name, *smmu, *this); + proc->beginTransaction(SMMUTranslRequest::fromPacket(pkt, ats_request)); + + smmu->runProcessTiming(proc, pkt); + + return true; +} + +bool +SMMUv3SlaveInterface::atsMasterRecvTimingResp(PacketPtr pkt) +{ + DPRINTF(SMMUv3, "[t] ATS master resp addr=%#x size=%#x\n", + pkt->getAddr(), pkt->getSize()); + + // @todo: We need to pay for this and not just zero it out + pkt->headerDelay = pkt->payloadDelay = 0; + + SMMUProcess *proc = + safe_cast(pkt->popSenderState()); + + smmu->runProcessTiming(proc, pkt); + + return true; +} + +void +SMMUv3SlaveInterface::sendDeviceRetry() +{ + slavePort->sendRetryReq(); +} + +void +SMMUv3SlaveInterface::atsSendDeviceRetry() +{ + DPRINTF(SMMUv3, "ATS retry\n"); + atsSlavePort.sendRetryReq(); +} + +void +SMMUv3SlaveInterface::scheduleDeviceRetry() +{ + if (deviceNeedsRetry && !sendDeviceRetryEvent.scheduled()) { + DPRINTF(SMMUv3, "sched slave retry\n"); + deviceNeedsRetry = false; + schedule(sendDeviceRetryEvent, nextCycle()); + } +} + +SMMUv3SlaveInterface* +SMMUv3SlaveInterfaceParams::create() +{ + return new SMMUv3SlaveInterface(this); +} diff --git a/src/dev/arm/smmu_v3_slaveifc.hh b/src/dev/arm/smmu_v3_slaveifc.hh new file mode 100644 index 000000000..a782ff9b7 --- /dev/null +++ b/src/dev/arm/smmu_v3_slaveifc.hh @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_SLAVEIFC_HH__ +#define __DEV_ARM_SMMU_V3_SLAVEIFC_HH__ + +#include + +#include "dev/arm/smmu_v3_caches.hh" +#include "dev/arm/smmu_v3_defs.hh" +#include "dev/arm/smmu_v3_events.hh" +#include "dev/arm/smmu_v3_ports.hh" +#include "dev/arm/smmu_v3_proc.hh" +#include "mem/mem_object.hh" +#include "params/SMMUv3SlaveInterface.hh" + +class SMMUTranslationProcess; +class SMMUv3; +class SMMUSlavePort; + +class SMMUv3SlaveInterface : public MemObject +{ + public: + SMMUv3 *smmu; + SMMUTLB* microTLB; + SMMUTLB* mainTLB; + + const bool microTLBEnable; + const bool mainTLBEnable; + + SMMUSemaphore slavePortSem; + SMMUSemaphore microTLBSem; + SMMUSemaphore mainTLBSem; + + const Cycles microTLBLat; + const Cycles mainTLBLat; + + SMMUSlavePort *slavePort; + SMMUATSSlavePort atsSlavePort; + SMMUATSMasterPort atsMasterPort; + + // in bytes + const unsigned portWidth; + + unsigned wrBufSlotsRemaining; + unsigned xlateSlotsRemaining; + + const bool prefetchEnable; + const bool prefetchReserveLastWay; + + std::list duplicateReqs; + SMMUSignal duplicateReqRemoved; + + std::list dependentReads[SMMU_MAX_TRANS_ID]; + std::list dependentWrites[SMMU_MAX_TRANS_ID]; + SMMUSignal dependentReqRemoved; + + // Receiving translation requests from the master device + Tick recvAtomic(PacketPtr pkt); + bool recvTimingReq(PacketPtr pkt); + void schedTimingResp(PacketPtr pkt); + + Tick atsSlaveRecvAtomic(PacketPtr pkt); + bool atsSlaveRecvTimingReq(PacketPtr pkt); + bool atsMasterRecvTimingResp(PacketPtr pkt); + void schedAtsTimingResp(PacketPtr pkt); + + void scheduleDeviceRetry(); + void sendDeviceRetry(); + void atsSendDeviceRetry(); + + bool deviceNeedsRetry; + bool atsDeviceNeedsRetry; + + SMMUDeviceRetryEvent sendDeviceRetryEvent; + EventWrapper< + SMMUv3SlaveInterface, + &SMMUv3SlaveInterface::atsSendDeviceRetry> atsSendDeviceRetryEvent; + + Port& getPort(const std::string &name, PortID id); + + public: + SMMUv3SlaveInterface(const SMMUv3SlaveInterfaceParams *p); + + ~SMMUv3SlaveInterface() + { + delete microTLB; + delete mainTLB; + } + + void setSMMU(SMMUv3 *_smmu) { smmu = _smmu; } + void sendRange(); +}; + +#endif /* __DEV_ARM_SMMU_V3_SLAVEIFC_HH__ */ diff --git a/src/dev/arm/smmu_v3_transl.cc b/src/dev/arm/smmu_v3_transl.cc new file mode 100644 index 000000000..4a3efc6d8 --- /dev/null +++ b/src/dev/arm/smmu_v3_transl.cc @@ -0,0 +1,1443 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#include "dev/arm/smmu_v3_transl.hh" + +#include "debug/SMMUv3.hh" +#include "debug/SMMUv3Hazard.hh" +#include "dev/arm/amba.hh" +#include "dev/arm/smmu_v3.hh" +#include "sim/system.hh" + +SMMUTranslRequest +SMMUTranslRequest::fromPacket(PacketPtr pkt, bool ats) +{ + SMMUTranslRequest req; + req.addr = pkt->getAddr(); + req.size = pkt->getSize(); + req.sid = pkt->req->streamId(); + req.ssid = pkt->req->hasSubstreamId() ? + pkt->req->substreamId() : 0; + req.isWrite = pkt->isWrite(); + req.isPrefetch = false; + req.isAtsRequest = ats; + req.pkt = pkt; + + return req; +} + +SMMUTranslRequest +SMMUTranslRequest::prefetch(Addr addr, uint32_t sid, uint32_t ssid) +{ + SMMUTranslRequest req; + req.addr = addr; + req.size = 0; + req.sid = sid; + req.ssid = ssid; + req.isWrite = false; + req.isPrefetch = true; + req.isAtsRequest = false; + req.pkt = NULL; + + return req; +} + +void +SMMUTranslationProcess::beginTransaction(const SMMUTranslRequest &req) +{ + request = req; + + reinit(); +} + +void +SMMUTranslationProcess::resumeTransaction() +{ + assert(smmu.system.isTimingMode()); + + assert(!"Stalls are broken"); + + Tick resumeTick = curTick(); + + (void) resumeTick; + DPRINTF(SMMUv3, "Resume at tick = %d. Fault duration = %d (%.3fus)\n", + resumeTick, resumeTick-faultTick, (resumeTick-faultTick) / 1e6); + + beginTransaction(request); + + smmu.runProcessTiming(this, request.pkt); +} + +void +SMMUTranslationProcess::main(Yield &yield) +{ + // Hack: + // The coroutine starts running as soon as it's created. + // But we need to wait for request data esp. in atomic mode. + SMMUAction a; + a.type = ACTION_INITIAL_NOP; + a.pkt = NULL; + yield(a); + + const Addr next4k = (request.addr + 0x1000ULL) & ~0xfffULL; + + if ((request.addr + request.size) > next4k) + panic("Transaction crosses 4k boundary (addr=%#x size=%#x)!\n", + request.addr, request.size); + + + unsigned numSlaveBeats = request.isWrite ? + (request.size + (ifc.portWidth - 1)) / ifc.portWidth : 1; + + doSemaphoreDown(yield, ifc.slavePortSem); + doDelay(yield, Cycles(numSlaveBeats)); + doSemaphoreUp(ifc.slavePortSem); + + + recvTick = curTick(); + + + if (!(smmu.regs.cr0 & 0x1)) { + // SMMU disabled + doDelay(yield, Cycles(1)); + completeTransaction(yield, bypass(request.addr)); + return; + } + + TranslResult tr; + bool wasPrefetched = false; + + if (request.isPrefetch) { + // Abort prefetch if: + // - there's already a transaction looking up the same 4k page, OR + // - requested address is already in the TLB. + if (hazard4kCheck() || ifcTLBLookup(yield, tr, wasPrefetched)) + completePrefetch(yield); // this never returns + + hazard4kRegister(); + + tr = smmuTranslation(yield); + + if (tr.fault == FAULT_NONE) + ifcTLBUpdate(yield, tr); + + hazard4kRelease(); + + completePrefetch(yield); + } else { + hazardIdRegister(); + + if (!microTLBLookup(yield, tr)) { + bool hit = ifcTLBLookup(yield, tr, wasPrefetched); + if (!hit) { + while (!hit && hazard4kCheck()) { + hazard4kHold(yield); + hit = ifcTLBLookup(yield, tr, wasPrefetched); + } + } + + // Issue prefetch if: + // - there was a TLB hit and the entry was prefetched, OR + // - TLB miss was successfully serviced + if (hit) { + if (wasPrefetched) + issuePrefetch(next4k); + } else { + hazard4kRegister(); + + tr = smmuTranslation(yield); + + if (tr.fault == FAULT_NONE) { + ifcTLBUpdate(yield, tr); + + issuePrefetch(next4k); + } + + hazard4kRelease(); + } + + if (tr.fault == FAULT_NONE) + microTLBUpdate(yield, tr); + } + + hazardIdHold(yield); + hazardIdRelease(); + + if (tr.fault != FAULT_NONE) + panic("fault\n"); + + completeTransaction(yield, tr); + } +} + +SMMUTranslationProcess::TranslResult +SMMUTranslationProcess::bypass(Addr addr) const +{ + TranslResult tr; + tr.fault = FAULT_NONE; + tr.addr = addr; + tr.addrMask = 0; + tr.writable = 1; + + return tr; +} + +SMMUTranslationProcess::TranslResult +SMMUTranslationProcess::smmuTranslation(Yield &yield) +{ + TranslResult tr; + + // Need SMMU credit to proceed + doSemaphoreDown(yield, smmu.transSem); + + // Simulate pipelined IFC->SMMU link + doSemaphoreDown(yield, smmu.ifcSmmuSem); + doDelay(yield, Cycles(1)); // serialize transactions + doSemaphoreUp(smmu.ifcSmmuSem); + doDelay(yield, smmu.ifcSmmuLat - Cycles(1)); // remaining pipeline delay + + bool haveConfig = true; + if (!configCacheLookup(yield, context)) { + if(findConfig(yield, context, tr)) { + configCacheUpdate(yield, context); + } else { + haveConfig = false; + } + } + + if (haveConfig && !smmuTLBLookup(yield, tr)) { + // SMMU main TLB miss + + // Need PTW slot to proceed + doSemaphoreDown(yield, smmu.ptwSem); + + // Page table walk + Tick ptwStartTick = curTick(); + + if (context.stage1Enable) { + tr = translateStage1And2(yield, request.addr); + } else if (context.stage2Enable) { + tr = translateStage2(yield, request.addr, true); + } else { + tr = bypass(request.addr); + } + + if (context.stage1Enable || context.stage2Enable) + smmu.ptwTimeDist.sample(curTick() - ptwStartTick); + + // Free PTW slot + doSemaphoreUp(smmu.ptwSem); + + if (tr.fault == FAULT_NONE) + smmuTLBUpdate(yield, tr); + } + + // Simulate pipelined SMMU->SLAVE INTERFACE link + doSemaphoreDown(yield, smmu.smmuIfcSem); + doDelay(yield, Cycles(1)); // serialize transactions + doSemaphoreUp(smmu.smmuIfcSem); + doDelay(yield, smmu.smmuIfcLat - Cycles(1)); // remaining pipeline delay + + // return SMMU credit + doSemaphoreUp(smmu.transSem); + + return tr; +} + +bool +SMMUTranslationProcess::microTLBLookup(Yield &yield, TranslResult &tr) +{ + if (!ifc.microTLBEnable) + return false; + + doSemaphoreDown(yield, ifc.microTLBSem); + doDelay(yield, ifc.microTLBLat); + const SMMUTLB::Entry *e = + ifc.microTLB->lookup(request.sid, request.ssid, request.addr); + doSemaphoreUp(ifc.microTLBSem); + + if (!e) { + DPRINTF(SMMUv3, "micro TLB miss vaddr=%#x sid=%#x ssid=%#x\n", + request.addr, request.sid, request.ssid); + + return false; + } + + DPRINTF(SMMUv3, + "micro TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x paddr=%#x\n", + request.addr, e->vaMask, request.sid, request.ssid, e->pa); + + tr.fault = FAULT_NONE; + tr.addr = e->pa + (request.addr & ~e->vaMask);; + tr.addrMask = e->vaMask; + tr.writable = e->permissions; + + return true; +} + +bool +SMMUTranslationProcess::ifcTLBLookup(Yield &yield, TranslResult &tr, + bool &wasPrefetched) +{ + if (!ifc.mainTLBEnable) + return false; + + doSemaphoreDown(yield, ifc.mainTLBSem); + doDelay(yield, ifc.mainTLBLat); + const SMMUTLB::Entry *e = + ifc.mainTLB->lookup(request.sid, request.ssid, request.addr); + doSemaphoreUp(ifc.mainTLBSem); + + if (!e) { + DPRINTF(SMMUv3, + "SLAVE Interface TLB miss vaddr=%#x sid=%#x ssid=%#x\n", + request.addr, request.sid, request.ssid); + + return false; + } + + DPRINTF(SMMUv3, + "SLAVE Interface TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x " + "paddr=%#x\n", request.addr, e->vaMask, request.sid, + request.ssid, e->pa); + + tr.fault = FAULT_NONE; + tr.addr = e->pa + (request.addr & ~e->vaMask);; + tr.addrMask = e->vaMask; + tr.writable = e->permissions; + wasPrefetched = e->prefetched; + + return true; +} + +bool +SMMUTranslationProcess::smmuTLBLookup(Yield &yield, TranslResult &tr) +{ + if (!smmu.tlbEnable) + return false; + + doSemaphoreDown(yield, smmu.tlbSem); + doDelay(yield, smmu.tlbLat); + const ARMArchTLB::Entry *e = + smmu.tlb.lookup(request.addr, context.asid, context.vmid); + doSemaphoreUp(smmu.tlbSem); + + if (!e) { + DPRINTF(SMMUv3, "SMMU TLB miss vaddr=%#x asid=%#x vmid=%#x\n", + request.addr, context.asid, context.vmid); + + return false; + } + + DPRINTF(SMMUv3, + "SMMU TLB hit vaddr=%#x amask=%#x asid=%#x vmid=%#x paddr=%#x\n", + request.addr, e->vaMask, context.asid, context.vmid, e->pa); + + tr.fault = FAULT_NONE; + tr.addr = e->pa + (request.addr & ~e->vaMask);; + tr.addrMask = e->vaMask; + tr.writable = e->permissions; + + return true; +} + +void +SMMUTranslationProcess::microTLBUpdate(Yield &yield, + const TranslResult &tr) +{ + assert(tr.fault == FAULT_NONE); + + if (!ifc.microTLBEnable) + return; + + SMMUTLB::Entry e; + e.valid = true; + e.prefetched = false; + e.sid = request.sid; + e.ssid = request.ssid; + e.vaMask = tr.addrMask; + e.va = request.addr & e.vaMask; + e.pa = tr.addr & e.vaMask; + e.permissions = tr.writable; + e.asid = context.asid; + e.vmid = context.vmid; + + doSemaphoreDown(yield, ifc.microTLBSem); + + DPRINTF(SMMUv3, + "micro TLB upd vaddr=%#x amask=%#x paddr=%#x sid=%#x ssid=%#x\n", + e.va, e.vaMask, e.pa, e.sid, e.ssid); + + ifc.microTLB->store(e, SMMUTLB::ALLOC_ANY_WAY); + + doSemaphoreUp(ifc.microTLBSem); +} + +void +SMMUTranslationProcess::ifcTLBUpdate(Yield &yield, + const TranslResult &tr) +{ + assert(tr.fault == FAULT_NONE); + + if (!ifc.mainTLBEnable) + return; + + SMMUTLB::Entry e; + e.valid = true; + e.prefetched = request.isPrefetch; + e.sid = request.sid; + e.ssid = request.ssid; + e.vaMask = tr.addrMask; + e.va = request.addr & e.vaMask; + e.pa = tr.addr & e.vaMask; + e.permissions = tr.writable; + e.asid = context.asid; + e.vmid = context.vmid; + + SMMUTLB::AllocPolicy alloc = SMMUTLB::ALLOC_ANY_WAY; + if (ifc.prefetchEnable && ifc.prefetchReserveLastWay) + alloc = request.isPrefetch ? + SMMUTLB::ALLOC_LAST_WAY : SMMUTLB::ALLOC_ANY_BUT_LAST_WAY; + + doSemaphoreDown(yield, ifc.mainTLBSem); + + DPRINTF(SMMUv3, + "SLAVE Interface upd vaddr=%#x amask=%#x paddr=%#x sid=%#x " + "ssid=%#x\n", e.va, e.vaMask, e.pa, e.sid, e.ssid); + + ifc.mainTLB->store(e, alloc); + + doSemaphoreUp(ifc.mainTLBSem); +} + +void +SMMUTranslationProcess::smmuTLBUpdate(Yield &yield, + const TranslResult &tr) +{ + assert(tr.fault == FAULT_NONE); + + if (!smmu.tlbEnable) + return; + + ARMArchTLB::Entry e; + e.valid = true; + e.vaMask = tr.addrMask; + e.va = request.addr & e.vaMask; + e.asid = context.asid; + e.vmid = context.vmid; + e.pa = tr.addr & e.vaMask; + e.permissions = tr.writable; + + doSemaphoreDown(yield, smmu.tlbSem); + + DPRINTF(SMMUv3, + "SMMU TLB upd vaddr=%#x amask=%#x paddr=%#x asid=%#x vmid=%#x\n", + e.va, e.vaMask, e.pa, e.asid, e.vmid); + + smmu.tlb.store(e); + + doSemaphoreUp(smmu.tlbSem); +} + +bool +SMMUTranslationProcess::configCacheLookup(Yield &yield, TranslContext &tc) +{ + if (!smmu.configCacheEnable) + return false; + + doSemaphoreDown(yield, smmu.configSem); + doDelay(yield, smmu.configLat); + const ConfigCache::Entry *e = + smmu.configCache.lookup(request.sid, request.ssid); + doSemaphoreUp(smmu.configSem); + + if (!e) { + DPRINTF(SMMUv3, "Config miss sid=%#x ssid=%#x\n", + request.sid, request.ssid); + + return false; + } + + DPRINTF(SMMUv3, "Config hit sid=%#x ssid=%#x ttb=%#08x asid=%#x\n", + request.sid, request.ssid, e->ttb0, e->asid); + + tc.stage1Enable = e->stage1_en; + tc.stage2Enable = e->stage2_en; + + tc.ttb0 = e->ttb0; + tc.ttb1 = e->ttb1; + tc.asid = e->asid; + tc.httb = e->httb; + tc.vmid = e->vmid; + + tc.stage1TranslGranule = e->stage1_tg; + tc.stage2TranslGranule = e->stage2_tg; + + return true; +} + +void +SMMUTranslationProcess::configCacheUpdate(Yield &yield, + const TranslContext &tc) +{ + if (!smmu.configCacheEnable) + return; + + ConfigCache::Entry e; + e.valid = true; + e.sid = request.sid; + e.ssid = request.ssid; + e.stage1_en = tc.stage1Enable; + e.stage2_en = tc.stage2Enable; + e.ttb0 = tc.ttb0; + e.ttb1 = tc.ttb1; + e.asid = tc.asid; + e.httb = tc.httb; + e.vmid = tc.vmid; + e.stage1_tg = tc.stage1TranslGranule; + e.stage2_tg = tc.stage2TranslGranule; + + doSemaphoreDown(yield, smmu.configSem); + + DPRINTF(SMMUv3, "Config upd sid=%#x ssid=%#x\n", e.sid, e.ssid); + + smmu.configCache.store(e); + + doSemaphoreUp(smmu.configSem); +} + +bool +SMMUTranslationProcess::findConfig(Yield &yield, + TranslContext &tc, + TranslResult &tr) +{ + tc.stage1Enable = false; + tc.stage2Enable = false; + + StreamTableEntry ste; + doReadSTE(yield, ste, request.sid); + + switch (ste.dw0.config) { + case STE_CONFIG_BYPASS: + break; + + case STE_CONFIG_STAGE1_ONLY: + tc.stage1Enable = true; + break; + + case STE_CONFIG_STAGE2_ONLY: + tc.stage2Enable = true; + break; + + case STE_CONFIG_STAGE1_AND_2: + tc.stage1Enable = true; + tc.stage2Enable = true; + break; + + default: + panic("Bad or unimplemented STE config %d\n", + ste.dw0.config); + } + + + // Establish stage 2 context first since + // Context Descriptors can be in IPA space. + if (tc.stage2Enable) { + tc.httb = ste.dw3.s2ttb << STE_S2TTB_SHIFT; + tc.vmid = ste.dw2.s2vmid; + tc.stage2TranslGranule = ste.dw2.s2tg; + } else { + tc.httb = 0xdeadbeef; + tc.vmid = 0; + tc.stage2TranslGranule = TRANS_GRANULE_INVALID; + } + + + // Now fetch stage 1 config. + if (context.stage1Enable) { + ContextDescriptor cd; + doReadCD(yield, cd, ste, request.sid, request.ssid); + + tc.ttb0 = cd.dw1.ttb0 << CD_TTB_SHIFT; + tc.ttb1 = cd.dw2.ttb1 << CD_TTB_SHIFT; + tc.asid = cd.dw0.asid; + tc.stage1TranslGranule = cd.dw0.tg0; + } else { + tc.ttb0 = 0xcafebabe; + tc.ttb1 = 0xcafed00d; + tc.asid = 0; + tc.stage1TranslGranule = TRANS_GRANULE_INVALID; + } + + return true; +} + +void +SMMUTranslationProcess::walkCacheLookup( + Yield &yield, + const WalkCache::Entry *&walkEntry, + Addr addr, uint16_t asid, uint16_t vmid, + unsigned stage, unsigned level) +{ + const char *indent = stage==2 ? " " : ""; + (void) indent; // this is only used in DPRINTFs + + const PageTableOps *pt_ops = + stage == 1 ? + smmu.getPageTableOps(context.stage1TranslGranule) : + smmu.getPageTableOps(context.stage2TranslGranule); + + unsigned walkCacheLevels = + smmu.walkCacheEnable ? + (stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels) : + 0; + + if ((1 << level) & walkCacheLevels) { + doSemaphoreDown(yield, smmu.walkSem); + doDelay(yield, smmu.walkLat); + + walkEntry = smmu.walkCache.lookup(addr, pt_ops->walkMask(level), + asid, vmid, stage, level); + + if (walkEntry) { + DPRINTF(SMMUv3, "%sWalkCache hit va=%#x asid=%#x vmid=%#x " + "base=%#x (S%d, L%d)\n", + indent, addr, asid, vmid, walkEntry->pa, stage, level); + } else { + DPRINTF(SMMUv3, "%sWalkCache miss va=%#x asid=%#x vmid=%#x " + "(S%d, L%d)\n", + indent, addr, asid, vmid, stage, level); + } + + doSemaphoreUp(smmu.walkSem); + } +} + +void +SMMUTranslationProcess::walkCacheUpdate(Yield &yield, Addr va, + Addr vaMask, Addr pa, + unsigned stage, unsigned level, + bool leaf, uint8_t permissions) +{ + unsigned walkCacheLevels = + stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels; + + if (smmu.walkCacheEnable && ((1<lastLevel(); level++) { + Addr pte_addr = walkPtr + pt_ops->index(addr, level); + + DPRINTF(SMMUv3, "Fetching S1 L%d PTE from pa=%#08x\n", + level, pte_addr); + + doReadPTE(yield, addr, pte_addr, &pte, 1, level); + + DPRINTF(SMMUv3, "Got S1 L%d PTE=%#x from pa=%#08x\n", + level, pte, pte_addr); + + doSemaphoreDown(yield, smmu.cycleSem); + doDelay(yield, Cycles(1)); + doSemaphoreUp(smmu.cycleSem); + + bool valid = pt_ops->isValid(pte, level); + bool leaf = pt_ops->isLeaf(pte, level); + + if (!valid) { + DPRINTF(SMMUv3, "S1 PTE not valid - fault\n"); + + TranslResult tr; + tr.fault = FAULT_TRANSLATION; + return tr; + } + + if (valid && leaf && request.isWrite && + !pt_ops->isWritable(pte, level, false)) + { + DPRINTF(SMMUv3, "S1 page not writable - fault\n"); + + TranslResult tr; + tr.fault = FAULT_PERMISSION; + return tr; + } + + walkPtr = pt_ops->nextLevelPointer(pte, level); + + if (leaf) + break; + + if (context.stage2Enable) { + TranslResult s2tr = translateStage2(yield, walkPtr, false); + if (s2tr.fault != FAULT_NONE) + return s2tr; + + walkPtr = s2tr.addr; + } + + walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr, + 1, level, leaf, 0); + } + + TranslResult tr; + tr.fault = FAULT_NONE; + tr.addrMask = pt_ops->pageMask(pte, level); + tr.addr = walkPtr + (addr & ~tr.addrMask); + tr.writable = pt_ops->isWritable(pte, level, false); + + if (context.stage2Enable) { + TranslResult s2tr = translateStage2(yield, tr.addr, true); + if (s2tr.fault != FAULT_NONE) + return s2tr; + + tr = combineTranslations(tr, s2tr); + } + + walkCacheUpdate(yield, addr, tr.addrMask, tr.addr, + 1, level, true, tr.writable); + + return tr; +} + +SMMUTranslationProcess::TranslResult +SMMUTranslationProcess::walkStage2(Yield &yield, Addr addr, bool final_tr, + const PageTableOps *pt_ops, + unsigned level, Addr walkPtr) +{ + PageTableOps::pte_t pte; + + doSemaphoreDown(yield, smmu.cycleSem); + doDelay(yield, Cycles(1)); + doSemaphoreUp(smmu.cycleSem); + + for (; level <= pt_ops->lastLevel(); level++) { + Addr pte_addr = walkPtr + pt_ops->index(addr, level); + + DPRINTF(SMMUv3, " Fetching S2 L%d PTE from pa=%#08x\n", + level, pte_addr); + + doReadPTE(yield, addr, pte_addr, &pte, 2, level); + + DPRINTF(SMMUv3, " Got S2 L%d PTE=%#x from pa=%#08x\n", + level, pte, pte_addr); + + doSemaphoreDown(yield, smmu.cycleSem); + doDelay(yield, Cycles(1)); + doSemaphoreUp(smmu.cycleSem); + + bool valid = pt_ops->isValid(pte, level); + bool leaf = pt_ops->isLeaf(pte, level); + + if (!valid) { + DPRINTF(SMMUv3, " S2 PTE not valid - fault\n"); + + TranslResult tr; + tr.fault = FAULT_TRANSLATION; + return tr; + } + + if (valid && leaf && request.isWrite && + !pt_ops->isWritable(pte, level, true)) + { + DPRINTF(SMMUv3, " S2 PTE not writable = fault\n"); + + TranslResult tr; + tr.fault = FAULT_PERMISSION; + return tr; + } + + walkPtr = pt_ops->nextLevelPointer(pte, level); + + if (final_tr || smmu.walkCacheNonfinalEnable) + walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr, + 2, level, leaf, + leaf ? pt_ops->isWritable(pte, level, true) : 0); + if (leaf) + break; + } + + TranslResult tr; + tr.fault = FAULT_NONE; + tr.addrMask = pt_ops->pageMask(pte, level); + tr.addr = walkPtr + (addr & ~tr.addrMask); + tr.writable = pt_ops->isWritable(pte, level, true); + + return tr; +} + +SMMUTranslationProcess::TranslResult +SMMUTranslationProcess::translateStage1And2(Yield &yield, Addr addr) +{ + const PageTableOps *pt_ops = + smmu.getPageTableOps(context.stage1TranslGranule); + + const WalkCache::Entry *walk_ep = NULL; + unsigned level; + + // Level here is actually (level+1) so we can count down + // to 0 using unsigned int. + for (level = pt_ops->lastLevel() + 1; + level > pt_ops->firstLevel(); + level--) + { + walkCacheLookup(yield, walk_ep, addr, + context.asid, context.vmid, 1, level-1); + + if (walk_ep) + break; + } + + // Correct level (see above). + level -= 1; + + TranslResult tr; + if (walk_ep) { + if (walk_ep->leaf) { + tr.fault = FAULT_NONE; + tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); + tr.addrMask = walk_ep->vaMask; + tr.writable = walk_ep->permissions; + } else { + tr = walkStage1And2(yield, addr, pt_ops, level+1, walk_ep->pa); + } + } else { + Addr table_addr = context.ttb0; + if (context.stage2Enable) { + TranslResult s2tr = translateStage2(yield, table_addr, false); + if (s2tr.fault != FAULT_NONE) + return s2tr; + + table_addr = s2tr.addr; + } + + tr = walkStage1And2(yield, addr, pt_ops, pt_ops->firstLevel(), + table_addr); + } + + if (tr.fault == FAULT_NONE) + DPRINTF(SMMUv3, "Translated vaddr %#x to paddr %#x\n", addr, tr.addr); + + return tr; +} + +SMMUTranslationProcess::TranslResult +SMMUTranslationProcess::translateStage2(Yield &yield, Addr addr, bool final_tr) +{ + const PageTableOps *pt_ops = + smmu.getPageTableOps(context.stage2TranslGranule); + + const IPACache::Entry *ipa_ep = NULL; + if (smmu.ipaCacheEnable) { + doSemaphoreDown(yield, smmu.ipaSem); + doDelay(yield, smmu.ipaLat); + ipa_ep = smmu.ipaCache.lookup(addr, context.vmid); + doSemaphoreUp(smmu.ipaSem); + } + + if (ipa_ep) { + TranslResult tr; + tr.fault = FAULT_NONE; + tr.addr = ipa_ep->pa + (addr & ~ipa_ep->ipaMask); + tr.addrMask = ipa_ep->ipaMask; + tr.writable = ipa_ep->permissions; + + DPRINTF(SMMUv3, " IPACache hit ipa=%#x vmid=%#x pa=%#x\n", + addr, context.vmid, tr.addr); + + return tr; + } else if (smmu.ipaCacheEnable) { + DPRINTF(SMMUv3, " IPACache miss ipa=%#x vmid=%#x\n", + addr, context.vmid); + } + + const WalkCache::Entry *walk_ep = NULL; + unsigned level = pt_ops->firstLevel(); + + if (final_tr || smmu.walkCacheNonfinalEnable) { + // Level here is actually (level+1) so we can count down + // to 0 using unsigned int. + for (level = pt_ops->lastLevel() + 1; + level > pt_ops->firstLevel(); + level--) + { + walkCacheLookup(yield, walk_ep, addr, + 0, context.vmid, 2, level-1); + + if (walk_ep) + break; + } + + // Correct level (see above). + level -= 1; + } + + TranslResult tr; + if (walk_ep) { + if (walk_ep->leaf) { + tr.fault = FAULT_NONE; + tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); + tr.addrMask = walk_ep->vaMask; + tr.writable = walk_ep->permissions; + } else { + tr = walkStage2(yield, addr, final_tr, pt_ops, + level + 1, walk_ep->pa); + } + } else { + tr = walkStage2(yield, addr, final_tr, pt_ops, pt_ops->firstLevel(), + context.httb); + } + + if (tr.fault == FAULT_NONE) + DPRINTF(SMMUv3, " Translated %saddr %#x to paddr %#x\n", + context.stage1Enable ? "ip" : "v", addr, tr.addr); + + if (smmu.ipaCacheEnable) { + IPACache::Entry e; + e.valid = true; + e.ipaMask = tr.addrMask; + e.ipa = addr & e.ipaMask; + e.pa = tr.addr & tr.addrMask; + e.permissions = tr.writable; + e.vmid = context.vmid; + + doSemaphoreDown(yield, smmu.ipaSem); + smmu.ipaCache.store(e); + doSemaphoreUp(smmu.ipaSem); + } + + return tr; +} + +SMMUTranslationProcess::TranslResult +SMMUTranslationProcess::combineTranslations(const TranslResult &s1tr, + const TranslResult &s2tr) const +{ + if (s2tr.fault != FAULT_NONE) + return s2tr; + + assert(s1tr.fault == FAULT_NONE); + + TranslResult tr; + tr.fault = FAULT_NONE; + tr.addr = s2tr.addr; + tr.addrMask = s1tr.addrMask | s2tr.addrMask; + tr.writable = s1tr.writable & s2tr.writable; + + return tr; +} + +bool +SMMUTranslationProcess::hazard4kCheck() +{ + Addr addr4k = request.addr & ~0xfffULL; + + for (auto it = ifc.duplicateReqs.begin(); + it != ifc.duplicateReqs.end(); + ++it) + { + Addr other4k = (*it)->request.addr & ~0xfffULL; + if (addr4k == other4k) + return true; + } + + return false; +} + +void +SMMUTranslationProcess::hazard4kRegister() +{ + DPRINTF(SMMUv3Hazard, "4kReg: p=%p a4k=%#x\n", + this, request.addr & ~0xfffULL); + + ifc.duplicateReqs.push_back(this); +} + +void +SMMUTranslationProcess::hazard4kHold(Yield &yield) +{ + Addr addr4k = request.addr & ~0xfffULL; + + bool found_hazard; + + do { + found_hazard = false; + + for (auto it = ifc.duplicateReqs.begin(); + it!=ifc.duplicateReqs.end() && *it!=this; + ++it) + { + Addr other4k = (*it)->request.addr & ~0xfffULL; + + DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x Q: p=%p a4k=%#x\n", + this, addr4k, *it, other4k); + + if (addr4k == other4k) { + DPRINTF(SMMUv3Hazard, + "4kHold: p=%p a4k=%#x WAIT on p=%p a4k=%#x\n", + this, addr4k, *it, other4k); + + doWaitForSignal(yield, ifc.duplicateReqRemoved); + + DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x RESUME\n", + this, addr4k); + + // This is to avoid checking *it!=this after doWaitForSignal() + // since it could have been deleted. + found_hazard = true; + break; + } + } + } while (found_hazard); +} + +void +SMMUTranslationProcess::hazard4kRelease() +{ + DPRINTF(SMMUv3Hazard, "4kRel: p=%p a4k=%#x\n", + this, request.addr & ~0xfffULL); + + std::list::iterator it; + + for (it = ifc.duplicateReqs.begin(); it != ifc.duplicateReqs.end(); ++it) + if (*it == this) + break; + + if (it == ifc.duplicateReqs.end()) + panic("hazard4kRelease: request not found"); + + ifc.duplicateReqs.erase(it); + + doBroadcastSignal(ifc.duplicateReqRemoved); +} + +void +SMMUTranslationProcess::hazardIdRegister() +{ + auto orderId = AMBA::orderId(request.pkt); + + DPRINTF(SMMUv3Hazard, "IdReg: p=%p oid=%d\n", this, orderId); + + assert(orderId < SMMU_MAX_TRANS_ID); + + std::list &depReqs = + request.isWrite ? + ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; + depReqs.push_back(this); +} + +void +SMMUTranslationProcess::hazardIdHold(Yield &yield) +{ + auto orderId = AMBA::orderId(request.pkt); + + DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d\n", this, orderId); + + std::list &depReqs = + request.isWrite ? + ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; + std::list::iterator it; + + bool found_hazard; + + do { + found_hazard = false; + + for (auto it = depReqs.begin(); it!=depReqs.end() && *it!=this; ++it) { + DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d Q: %p\n", + this, orderId, *it); + + if (AMBA::orderId((*it)->request.pkt) == orderId) { + DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d WAIT on=%p\n", + this, orderId, *it); + + doWaitForSignal(yield, ifc.dependentReqRemoved); + + DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d RESUME\n", + this, orderId); + + // This is to avoid checking *it!=this after doWaitForSignal() + // since it could have been deleted. + found_hazard = true; + break; + } + } + } while (found_hazard); +} + +void +SMMUTranslationProcess::hazardIdRelease() +{ + auto orderId = AMBA::orderId(request.pkt); + + DPRINTF(SMMUv3Hazard, "IdRel: p=%p oid=%d\n", this, orderId); + + std::list &depReqs = + request.isWrite ? + ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; + std::list::iterator it; + + for (it = depReqs.begin(); it != depReqs.end(); ++it) { + if (*it == this) + break; + } + + if (it == depReqs.end()) + panic("hazardIdRelease: request not found"); + + depReqs.erase(it); + + doBroadcastSignal(ifc.dependentReqRemoved); +} + +void +SMMUTranslationProcess::issuePrefetch(Addr addr) +{ + if (!smmu.system.isTimingMode()) + return; + + if (!ifc.prefetchEnable || ifc.xlateSlotsRemaining == 0) + return; + + ifc.xlateSlotsRemaining--; + + std::string proc_name = csprintf("%sprf", name()); + SMMUTranslationProcess *proc = + new SMMUTranslationProcess(proc_name, smmu, ifc); + + proc->beginTransaction( + SMMUTranslRequest::prefetch(addr, request.sid, request.ssid)); + proc->scheduleWakeup(smmu.clockEdge(Cycles(1))); +} + +void +SMMUTranslationProcess::completeTransaction(Yield &yield, + const TranslResult &tr) +{ + assert(tr.fault == FAULT_NONE); + + unsigned numMasterBeats = request.isWrite ? + (request.size + (smmu.masterPortWidth-1)) + / smmu.masterPortWidth : + 1; + + doSemaphoreDown(yield, smmu.masterPortSem); + doDelay(yield, Cycles(numMasterBeats)); + doSemaphoreUp(smmu.masterPortSem); + + + smmu.translationTimeDist.sample(curTick() - recvTick); + ifc.xlateSlotsRemaining++; + if (!request.isAtsRequest && request.isWrite) + ifc.wrBufSlotsRemaining += + (request.size + (ifc.portWidth-1)) / ifc.portWidth; + + smmu.scheduleSlaveRetries(); + + + SMMUAction a; + + if (request.isAtsRequest) { + a.type = ACTION_SEND_RESP_ATS; + + if (smmu.system.isAtomicMode()) { + request.pkt->makeAtomicResponse(); + } else if (smmu.system.isTimingMode()) { + request.pkt->makeTimingResponse(); + } else { + panic("Not in atomic or timing mode"); + } + } else { + a.type = ACTION_SEND_REQ_FINAL; + a.ifc = &ifc; + } + + a.pkt = request.pkt; + a.delay = 0; + + a.pkt->setAddr(tr.addr); + a.pkt->req->setPaddr(tr.addr); + + yield(a); + + if (!request.isAtsRequest) { + PacketPtr pkt = yield.get(); + pkt->setAddr(request.addr); + + a.type = ACTION_SEND_RESP; + a.pkt = pkt; + a.ifc = &ifc; + a.delay = 0; + yield(a); + } +} + +void +SMMUTranslationProcess::completePrefetch(Yield &yield) +{ + ifc.xlateSlotsRemaining++; + + SMMUAction a; + a.type = ACTION_TERMINATE; + a.pkt = NULL; + a.ifc = &ifc; + a.delay = 0; + yield(a); +} + +void +SMMUTranslationProcess::sendEvent(Yield &yield, const SMMUEvent &ev) +{ + int sizeMask = mask(smmu.regs.eventq_base & Q_BASE_SIZE_MASK) & + Q_CONS_PROD_MASK; + + if (((smmu.regs.eventq_prod+1) & sizeMask) == + (smmu.regs.eventq_cons & sizeMask)) + panic("Event queue full - aborting\n"); + + Addr event_addr = + (smmu.regs.eventq_base & Q_BASE_ADDR_MASK) + + (smmu.regs.eventq_prod & sizeMask) * sizeof(ev); + + DPRINTF(SMMUv3, "Sending event to addr=%#08x (pos=%d): type=%#x stag=%#x " + "flags=%#x sid=%#x ssid=%#x va=%#08x ipa=%#x\n", + event_addr, smmu.regs.eventq_prod, ev.type, ev.stag, + ev.flags, ev.streamId, ev.substreamId, ev.va, ev.ipa); + + // This deliberately resets the overflow field in eventq_prod! + smmu.regs.eventq_prod = (smmu.regs.eventq_prod + 1) & sizeMask; + + doWrite(yield, event_addr, &ev, sizeof(ev)); + + if (!(smmu.regs.eventq_irq_cfg0 & E_BASE_ENABLE_MASK)) + panic("eventq msi not enabled\n"); + + doWrite(yield, smmu.regs.eventq_irq_cfg0 & E_BASE_ADDR_MASK, + &smmu.regs.eventq_irq_cfg1, sizeof(smmu.regs.eventq_irq_cfg1)); +} + +void +SMMUTranslationProcess::doReadSTE(Yield &yield, + StreamTableEntry &ste, + uint32_t sid) +{ + unsigned max_sid = 1 << (smmu.regs.strtab_base_cfg & ST_CFG_SIZE_MASK); + if (sid >= max_sid) + panic("SID %#x out of range, max=%#x", sid, max_sid); + + Addr ste_addr; + + if ((smmu.regs.strtab_base_cfg & ST_CFG_FMT_MASK) == ST_CFG_FMT_2LEVEL) { + unsigned split = + (smmu.regs.strtab_base_cfg & ST_CFG_SPLIT_MASK) >> ST_CFG_SPLIT_SHIFT; + + if (split!= 7 && split!=8 && split!=16) + panic("Invalid stream table split %d", split); + + uint64_t l2_ptr; + uint64_t l2_addr = + (smmu.regs.strtab_base & VMT_BASE_ADDR_MASK) + + bits(sid, 32, split) * sizeof(l2_ptr); + + DPRINTF(SMMUv3, "Read L1STE at %#x\n", l2_addr); + + doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, 0); + + DPRINTF(SMMUv3, "Got L1STE L1 at %#x: 0x%016x\n", l2_addr, l2_ptr); + + unsigned span = l2_ptr & ST_L2_SPAN_MASK; + if (span == 0) + panic("Invalid level 1 stream table descriptor"); + + unsigned index = bits(sid, split-1, 0); + if (index >= (1 << span)) + panic("StreamID %d out of level 1 descriptor range %d", + sid, 1<= max_ssid) + panic("SSID %#x out of range, max=%#x", ssid, max_ssid); + + if (ste.dw0.s1fmt==STAGE1_CFG_2L_4K || + ste.dw0.s1fmt==STAGE1_CFG_2L_64K) + { + unsigned split = ste.dw0.s1fmt==STAGE1_CFG_2L_4K ? 7 : 11; + + uint64_t l2_ptr; + uint64_t l2_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + + bits(ssid, 24, split) * sizeof(l2_ptr); + + if (context.stage2Enable) + l2_addr = translateStage2(yield, l2_addr, false).addr; + + DPRINTF(SMMUv3, "Read L1CD at %#x\n", l2_addr); + + doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, ssid); + + DPRINTF(SMMUv3, "Got L1CD at %#x: 0x%016x\n", l2_addr, l2_ptr); + + cd_addr = l2_ptr + bits(ssid, split-1, 0) * sizeof(cd); + + smmu.cdL1Fetches++; + } else if (ste.dw0.s1fmt == STAGE1_CFG_1L) { + cd_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + ssid*sizeof(cd); + } + } + + if (context.stage2Enable) + cd_addr = translateStage2(yield, cd_addr, false).addr; + + DPRINTF(SMMUv3, "Read CD at %#x\n", cd_addr); + + doReadConfig(yield, cd_addr, &cd, sizeof(cd), sid, ssid); + + DPRINTF(SMMUv3, "Got CD at %#x [0]: 0x%016x\n", cd_addr, cd.dw0); + DPRINTF(SMMUv3, " CD at %#x [1]: 0x%016x\n", cd_addr, cd.dw1); + DPRINTF(SMMUv3, " CD at %#x [2]: 0x%016x\n", cd_addr, cd.dw2); + DPRINTF(SMMUv3, " CD at %#x [3]: 0x%016x\n", cd_addr, cd.mair); + DPRINTF(SMMUv3, " CD at %#x [4]: 0x%016x\n", cd_addr, cd.amair); + DPRINTF(SMMUv3, " CD at %#x [5]: 0x%016x\n", cd_addr, cd._pad[0]); + DPRINTF(SMMUv3, " CD at %#x [6]: 0x%016x\n", cd_addr, cd._pad[1]); + DPRINTF(SMMUv3, " CD at %#x [7]: 0x%016x\n", cd_addr, cd._pad[2]); + + + if (!cd.dw0.valid) + panic("CD @ %#x not valid\n", cd_addr); + + smmu.cdFetches++; +} + +void +SMMUTranslationProcess::doReadConfig(Yield &yield, Addr addr, + void *ptr, size_t size, + uint32_t sid, uint32_t ssid) +{ + doRead(yield, addr, ptr, size); +} + +void +SMMUTranslationProcess::doReadPTE(Yield &yield, Addr va, Addr addr, + void *ptr, unsigned stage, + unsigned level) +{ + size_t pte_size = sizeof(PageTableOps::pte_t); + + Addr mask = pte_size - 1; + Addr base = addr & ~mask; + + doRead(yield, base, ptr, pte_size); +} diff --git a/src/dev/arm/smmu_v3_transl.hh b/src/dev/arm/smmu_v3_transl.hh new file mode 100644 index 000000000..ac0dc7777 --- /dev/null +++ b/src/dev/arm/smmu_v3_transl.hh @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2013, 2018-2019 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Authors: Stan Czerniawski + */ + +#ifndef __DEV_ARM_SMMU_V3_TRANSL_HH__ +#define __DEV_ARM_SMMU_V3_TRANSL_HH__ + +#include "dev/arm/smmu_v3_proc.hh" +#include "dev/arm/smmu_v3_ptops.hh" +#include "dev/arm/smmu_v3_slaveifc.hh" +#include "mem/packet.hh" + +struct SMMUTranslRequest +{ + Addr addr; + unsigned size; + uint32_t sid; // streamId + uint32_t ssid; // substreamId + bool isWrite; + bool isPrefetch; + bool isAtsRequest; + + PacketPtr pkt; + + static SMMUTranslRequest fromPacket(PacketPtr pkt, bool ats = false); + static SMMUTranslRequest prefetch(Addr addr, uint32_t sid, uint32_t ssid); +}; + +class SMMUTranslationProcess : public SMMUProcess +{ + private: + struct TranslContext + { + bool stage1Enable; + bool stage2Enable; + Addr ttb0, ttb1, httb; + uint16_t asid; + uint16_t vmid; + uint8_t stage1TranslGranule; + uint8_t stage2TranslGranule; + }; + + enum FaultType + { + FAULT_NONE, + FAULT_TRANSLATION, // F_TRANSLATION + FAULT_PERMISSION, // F_PERMISSION + }; + + struct TranslResult + { + FaultType fault; + Addr addr; + Addr addrMask; + bool writable; + }; + + SMMUv3SlaveInterface &ifc; + + SMMUTranslRequest request; + TranslContext context; + + Tick recvTick; + Tick faultTick; + + virtual void main(Yield &yield); + + TranslResult bypass(Addr addr) const; + TranslResult smmuTranslation(Yield &yield); + + bool microTLBLookup(Yield &yield, TranslResult &tr); + bool ifcTLBLookup(Yield &yield, TranslResult &tr, bool &wasPrefetched); + bool smmuTLBLookup(Yield &yield, TranslResult &tr); + + void microTLBUpdate(Yield &yield, const TranslResult &tr); + void ifcTLBUpdate(Yield &yield, const TranslResult &tr); + void smmuTLBUpdate(Yield &yield, const TranslResult &tr); + + bool configCacheLookup(Yield &yield, TranslContext &tc); + void configCacheUpdate(Yield &yield, const TranslContext &tc); + bool findConfig(Yield &yield, TranslContext &tc, TranslResult &tr); + + void walkCacheLookup(Yield &yield, + const WalkCache::Entry *&walkEntry, + Addr addr, uint16_t asid, uint16_t vmid, + unsigned stage, unsigned level); + + void walkCacheUpdate(Yield &yield, Addr va, Addr vaMask, Addr pa, + unsigned stage, unsigned level, + bool leaf, uint8_t permissions); + + TranslResult walkStage1And2(Yield &yield, Addr addr, + const PageTableOps *pt_ops, + unsigned level, Addr walkPtr); + + TranslResult walkStage2(Yield &yield, Addr addr, bool final_tr, + const PageTableOps *pt_ops, + unsigned level, Addr walkPtr); + + TranslResult translateStage1And2(Yield &yield, Addr addr); + TranslResult translateStage2(Yield &yield, Addr addr, bool final_tr); + + TranslResult combineTranslations(const TranslResult &s1tr, + const TranslResult &s2tr) const; + + /** + * Used to force ordering on transactions with same + * (SID, SSID, 4k page) to avoid multiple identical + * page-table walks. + */ + bool hazard4kCheck(); + void hazard4kRegister(); + void hazard4kHold(Yield &yield); + void hazard4kRelease(); + + /** + * Used to force ordering on transactions with the same orderId. + * This attempts to model AXI IDs. + */ + void hazardIdRegister(); + void hazardIdHold(Yield &yield); + void hazardIdRelease(); + + void issuePrefetch(Addr addr); + + void completeTransaction(Yield &yield, const TranslResult &tr); + void completePrefetch(Yield &yield); + + void sendEvent(Yield &yield, const SMMUEvent &ev); + + void doReadSTE(Yield &yield, StreamTableEntry &ste, uint32_t sid); + void doReadCD(Yield &yield, ContextDescriptor &cd, + const StreamTableEntry &ste, uint32_t sid, uint32_t ssid); + void doReadConfig(Yield &yield, Addr addr, void *ptr, size_t size, + uint32_t sid, uint32_t ssid); + void doReadPTE(Yield &yield, Addr va, Addr addr, void *ptr, + unsigned stage, unsigned level); + + public: + SMMUTranslationProcess(const std::string &name, SMMUv3 &_smmu, + SMMUv3SlaveInterface &_ifc) + : + SMMUProcess(name, _smmu), + ifc(_ifc) + { + reinit(); + } + + virtual ~SMMUTranslationProcess() {} + + void beginTransaction(const SMMUTranslRequest &req); + void resumeTransaction(); +}; + +#endif /* __DEV_ARM_SMMU_V3_TRANSL_HH__ */