From 14e349c729d8d0aae4950c1239f5a41304ad3706 Mon Sep 17 00:00:00 2001 From: Matthew Poremba Date: Fri, 17 Apr 2020 09:41:06 -0500 Subject: [PATCH] mem-ruby,mem-garnet: Multiple networks per RubySystem Add support for multiple networks per RubySystem. This is done by introducing local IDs to each network and translating from a global ID passed around through Ruby and SLICC code. The local IDs represents the NodeID of a MachineType in the network and are ordered the same way that NodeIDs are ordered using MachineType_base_number. If there are not multiple networks in a RubySystem the local and global IDs are the same value. This is useful in cases where multiple isolated networks are needed to support devices with Ruby caches which do not interact with other networks. For example, a dGPU device will have a cache hierarchy that will not interact with the CPU cache hierachy. Change-Id: I33a917b3a394eec84b16fbf001c3c2c44c047f66 JIRA: https://gem5.atlassian.net/browse/GEM5-445 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/27927 Reviewed-by: Jason Lowe-Power Reviewed-by: Bradford Beckmann Maintainer: Bradford Beckmann Tested-by: kokoro --- src/mem/ruby/network/Network.cc | 65 +++++++++++++++---- src/mem/ruby/network/Network.hh | 14 ++-- .../ruby/network/garnet2.0/GarnetNetwork.cc | 23 ++++--- src/mem/ruby/network/simple/SimpleNetwork.cc | 19 +++--- src/mem/ruby/system/RubySystem.cc | 17 +++-- src/mem/ruby/system/RubySystem.hh | 3 +- 6 files changed, 99 insertions(+), 42 deletions(-) diff --git a/src/mem/ruby/network/Network.cc b/src/mem/ruby/network/Network.cc index 57834f2e2..982b57e27 100644 --- a/src/mem/ruby/network/Network.cc +++ b/src/mem/ruby/network/Network.cc @@ -55,9 +55,35 @@ Network::Network(const Params *p) m_virtual_networks = p->number_of_virtual_networks; m_control_msg_size = p->control_msg_size; - // Total nodes/controllers in network + // Populate localNodeVersions with the version of each MachineType in + // this network. This will be used to compute a global to local ID. + // Do this by looking at the ext_node for each ext_link. There is one + // ext_node per ext_link and it points to an AbstractController. + // For RubySystems with one network global and local ID are the same. + std::unordered_map> localNodeVersions; + for (auto &it : params()->ext_links) { + AbstractController *cntrl = it->params()->ext_node; + localNodeVersions[cntrl->getType()].push_back(cntrl->getVersion()); + } + + // Compute a local ID for each MachineType using the same order as SLICC + NodeID local_node_id = 0; + for (int i = 0; i < MachineType_base_level(MachineType_NUM); ++i) { + MachineType mach = static_cast(i); + if (localNodeVersions.count(mach)) { + for (auto &ver : localNodeVersions.at(mach)) { + // Get the global ID Ruby will pass around + NodeID global_node_id = MachineType_base_number(mach) + ver; + globalToLocalMap.emplace(global_node_id, local_node_id); + ++local_node_id; + } + } + } + + // Total nodes/controllers in network is equal to the local node count // Must make sure this is called after the State Machine constructors - m_nodes = MachineType_base_number(MachineType_NUM); + m_nodes = local_node_id; + assert(m_nodes != 0); assert(m_virtual_networks != 0); @@ -158,11 +184,11 @@ Network::MessageSizeType_to_int(MessageSizeType size_type) } void -Network::checkNetworkAllocation(NodeID id, bool ordered, +Network::checkNetworkAllocation(NodeID local_id, bool ordered, int network_num, std::string vnet_type) { - fatal_if(id >= m_nodes, "Node ID is out of range"); + fatal_if(local_id >= m_nodes, "Node ID is out of range"); fatal_if(network_num >= m_virtual_networks, "Network id is out of range"); if (ordered) { @@ -174,25 +200,29 @@ Network::checkNetworkAllocation(NodeID id, bool ordered, void -Network::setToNetQueue(NodeID id, bool ordered, int network_num, +Network::setToNetQueue(NodeID global_id, bool ordered, int network_num, std::string vnet_type, MessageBuffer *b) { - checkNetworkAllocation(id, ordered, network_num, vnet_type); - while (m_toNetQueues[id].size() <= network_num) { - m_toNetQueues[id].push_back(nullptr); + NodeID local_id = getLocalNodeID(global_id); + checkNetworkAllocation(local_id, ordered, network_num, vnet_type); + + while (m_toNetQueues[local_id].size() <= network_num) { + m_toNetQueues[local_id].push_back(nullptr); } - m_toNetQueues[id][network_num] = b; + m_toNetQueues[local_id][network_num] = b; } void -Network::setFromNetQueue(NodeID id, bool ordered, int network_num, +Network::setFromNetQueue(NodeID global_id, bool ordered, int network_num, std::string vnet_type, MessageBuffer *b) { - checkNetworkAllocation(id, ordered, network_num, vnet_type); - while (m_fromNetQueues[id].size() <= network_num) { - m_fromNetQueues[id].push_back(nullptr); + NodeID local_id = getLocalNodeID(global_id); + checkNetworkAllocation(local_id, ordered, network_num, vnet_type); + + while (m_fromNetQueues[local_id].size() <= network_num) { + m_fromNetQueues[local_id].push_back(nullptr); } - m_fromNetQueues[id][network_num] = b; + m_fromNetQueues[local_id][network_num] = b; } NodeID @@ -212,3 +242,10 @@ Network::addressToNodeID(Addr addr, MachineType mtype) } return MachineType_base_count(mtype); } + +NodeID +Network::getLocalNodeID(NodeID global_id) const +{ + assert(globalToLocalMap.count(global_id)); + return globalToLocalMap.at(global_id); +} diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh index 606e6704b..bba0c5e7f 100644 --- a/src/mem/ruby/network/Network.hh +++ b/src/mem/ruby/network/Network.hh @@ -90,13 +90,13 @@ class Network : public ClockedObject static uint32_t MessageSizeType_to_int(MessageSizeType size_type); // returns the queue requested for the given component - void setToNetQueue(NodeID id, bool ordered, int netNumber, + void setToNetQueue(NodeID global_id, bool ordered, int netNumber, std::string vnet_type, MessageBuffer *b); - virtual void setFromNetQueue(NodeID id, bool ordered, int netNumber, + virtual void setFromNetQueue(NodeID global_id, bool ordered, int netNumber, std::string vnet_type, MessageBuffer *b); - virtual void checkNetworkAllocation(NodeID id, bool ordered, - int network_num, std::string vnet_type); + virtual void checkNetworkAllocation(NodeID local_id, bool ordered, + int network_num, std::string vnet_type); virtual void makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, const NetDest& routing_table_entry) = 0; @@ -140,6 +140,8 @@ class Network : public ClockedObject return RubyDummyPort::instance(); } + NodeID getLocalNodeID(NodeID global_id) const; + protected: // Private copy constructor and assignment operator Network(const Network& obj); @@ -182,6 +184,10 @@ class Network : public ClockedObject AddrRangeList ranges; }; std::unordered_multimap addrMap; + + // Global NodeID to local node map. If there are not multiple networks in + // the same RubySystem, this is a one-to-one mapping of global to local. + std::unordered_map globalToLocalMap; }; inline std::ostream& diff --git a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc index 8f226b1de..a88302b6c 100644 --- a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc +++ b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc @@ -146,10 +146,11 @@ GarnetNetwork::init() */ void -GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, +GarnetNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link, const NetDest& routing_table_entry) { - assert(src < m_nodes); + NodeID local_src = getLocalNodeID(global_src); + assert(local_src < m_nodes); GarnetExtLink* garnet_link = safe_cast(link); @@ -163,7 +164,7 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, PortDirection dst_inport_dirn = "Local"; m_routers[dest]->addInPort(dst_inport_dirn, net_link, credit_link); - m_nis[src]->addOutPort(net_link, credit_link, dest); + m_nis[local_src]->addOutPort(net_link, credit_link, dest); } /* @@ -173,10 +174,12 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, */ void -GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, - const NetDest& routing_table_entry) +GarnetNetwork::makeExtOutLink(SwitchID src, NodeID global_dest, + BasicLink* link, + const NetDest& routing_table_entry) { - assert(dest < m_nodes); + NodeID local_dest = getLocalNodeID(global_dest); + assert(local_dest < m_nodes); assert(src < m_routers.size()); assert(m_routers[src] != NULL); @@ -194,7 +197,7 @@ GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, m_routers[src]->addOutPort(src_outport_dirn, net_link, routing_table_entry, link->m_weight, credit_link); - m_nis[dest]->addInPort(net_link, credit_link); + m_nis[local_dest]->addInPort(net_link, credit_link); } /* @@ -233,9 +236,11 @@ GarnetNetwork::getNumRouters() // Get ID of router connected to a NI. int -GarnetNetwork::get_router_id(int ni) +GarnetNetwork::get_router_id(int global_ni) { - return m_nis[ni]->get_router_id(); + NodeID local_ni = getLocalNodeID(global_ni); + + return m_nis[local_ni]->get_router_id(); } void diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc index 84817e444..d3b551512 100644 --- a/src/mem/ruby/network/simple/SimpleNetwork.cc +++ b/src/mem/ruby/network/simple/SimpleNetwork.cc @@ -83,27 +83,30 @@ SimpleNetwork::init() // From a switch to an endpoint node void -SimpleNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, - const NetDest& routing_table_entry) +SimpleNetwork::makeExtOutLink(SwitchID src, NodeID global_dest, + BasicLink* link, + const NetDest& routing_table_entry) { - assert(dest < m_nodes); + NodeID local_dest = getLocalNodeID(global_dest); + assert(local_dest < m_nodes); assert(src < m_switches.size()); assert(m_switches[src] != NULL); SimpleExtLink *simple_link = safe_cast(link); - m_switches[src]->addOutPort(m_fromNetQueues[dest], routing_table_entry, - simple_link->m_latency, + m_switches[src]->addOutPort(m_fromNetQueues[local_dest], + routing_table_entry, simple_link->m_latency, simple_link->m_bw_multiplier); } // From an endpoint node to a switch void -SimpleNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, +SimpleNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link, const NetDest& routing_table_entry) { - assert(src < m_nodes); - m_switches[dest]->addInPort(m_toNetQueues[src]); + NodeID local_src = getLocalNodeID(global_src); + assert(local_src < m_nodes); + m_switches[dest]->addInPort(m_toNetQueues[local_src]); } // From a switch to a switch diff --git a/src/mem/ruby/system/RubySystem.cc b/src/mem/ruby/system/RubySystem.cc index 2aeff8cae..c9da20656 100644 --- a/src/mem/ruby/system/RubySystem.cc +++ b/src/mem/ruby/system/RubySystem.cc @@ -94,7 +94,7 @@ RubySystem::RubySystem(const Params *p) void RubySystem::registerNetwork(Network* network_ptr) { - m_network = network_ptr; + m_networks.emplace_back(network_ptr); } void @@ -108,7 +108,6 @@ RubySystem::registerAbstractController(AbstractController* cntrl) RubySystem::~RubySystem() { - delete m_network; delete m_profiler; } @@ -407,7 +406,9 @@ void RubySystem::resetStats() { m_start_cycle = curCycle(); - m_network->resetStats(); + for (auto& network : m_networks) { + network->resetStats(); + } } bool @@ -512,8 +513,10 @@ RubySystem::functionalRead(PacketPtr pkt) DPRINTF(RubySystem, "Network functionalRead lookup " "(num_maybe_stale=%d, num_busy = %d)\n", num_maybe_stale, num_busy); - if (m_network->functionalRead(pkt)) - return true; + for (auto& network : m_networks) { + if (network->functionalRead(pkt)) + return true; + } } return false; @@ -558,7 +561,9 @@ RubySystem::functionalWrite(PacketPtr pkt) } } - num_functional_writes += m_network->functionalWrite(pkt); + for (auto& network : m_networks) { + num_functional_writes += network->functionalWrite(pkt); + } DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes); return true; diff --git a/src/mem/ruby/system/RubySystem.hh b/src/mem/ruby/system/RubySystem.hh index 5d10991aa..240707285 100644 --- a/src/mem/ruby/system/RubySystem.hh +++ b/src/mem/ruby/system/RubySystem.hh @@ -130,7 +130,8 @@ class RubySystem : public ClockedObject SimpleMemory *m_phys_mem; const bool m_access_backing_store; - Network* m_network; + //std::vector m_networks; + std::vector> m_networks; std::vector m_abs_cntrl_vec; Cycles m_start_cycle; -- 2.30.2