mem-ruby,mem-garnet: Multiple networks per RubySystem

Add support for multiple networks per RubySystem. This is done by
introducing local IDs to each network and translating from a global ID
passed around through Ruby and SLICC code. The local IDs represents the
NodeID of a MachineType in the network and are ordered the same way
that NodeIDs are ordered using MachineType_base_number. If there are
not multiple networks in a RubySystem the local and global IDs are the
same value.

This is useful in cases where multiple isolated networks are needed to
support devices with Ruby caches which do not interact with other
networks. For example, a dGPU device will have a cache hierarchy that
will not interact with the CPU cache hierachy.

Change-Id: I33a917b3a394eec84b16fbf001c3c2c44c047f66
JIRA: https://gem5.atlassian.net/browse/GEM5-445
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/27927
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Reviewed-by: Bradford Beckmann <brad.beckmann@amd.com>
Maintainer: Bradford Beckmann <brad.beckmann@amd.com>
Tested-by: kokoro <noreply+kokoro@google.com>

diff --git a/src/mem/ruby/network/Network.cc b/src/mem/ruby/network/Network.cc
index 57834f2e2a589b38fcafecb85510b41f8390ac55..982b57e27b4c72a3e19122b77da1ffa0ccc2a95c 100644 (file)
--- a/src/mem/ruby/network/Network.cc
+++ b/src/mem/ruby/network/Network.cc
@@ -55,9 +55,35 @@ Network::Network(const Params *p)
     m_virtual_networks = p->number_of_virtual_networks;
     m_control_msg_size = p->control_msg_size;
 
-    // Total nodes/controllers in network
+    // Populate localNodeVersions with the version of each MachineType in
+    // this network. This will be used to compute a global to local ID.
+    // Do this by looking at the ext_node for each ext_link. There is one
+    // ext_node per ext_link and it points to an AbstractController.
+    // For RubySystems with one network global and local ID are the same.
+    std::unordered_map<MachineType, std::vector<NodeID>> localNodeVersions;
+    for (auto &it : params()->ext_links) {
+        AbstractController *cntrl = it->params()->ext_node;
+        localNodeVersions[cntrl->getType()].push_back(cntrl->getVersion());
+    }
+
+    // Compute a local ID for each MachineType using the same order as SLICC
+    NodeID local_node_id = 0;
+    for (int i = 0; i < MachineType_base_level(MachineType_NUM); ++i) {
+        MachineType mach = static_cast<MachineType>(i);
+        if (localNodeVersions.count(mach)) {
+            for (auto &ver : localNodeVersions.at(mach)) {
+                // Get the global ID Ruby will pass around
+                NodeID global_node_id = MachineType_base_number(mach) + ver;
+                globalToLocalMap.emplace(global_node_id, local_node_id);
+                ++local_node_id;
+            }
+        }
+    }
+
+    // Total nodes/controllers in network is equal to the local node count
     // Must make sure this is called after the State Machine constructors
-    m_nodes = MachineType_base_number(MachineType_NUM);
+    m_nodes = local_node_id;
+
     assert(m_nodes != 0);
     assert(m_virtual_networks != 0);
 
@@ -158,11 +184,11 @@ Network::MessageSizeType_to_int(MessageSizeType size_type)
 }
 
 void
-Network::checkNetworkAllocation(NodeID id, bool ordered,
+Network::checkNetworkAllocation(NodeID local_id, bool ordered,
                                         int network_num,
                                         std::string vnet_type)
 {
-    fatal_if(id >= m_nodes, "Node ID is out of range");
+    fatal_if(local_id >= m_nodes, "Node ID is out of range");
     fatal_if(network_num >= m_virtual_networks, "Network id is out of range");
 
     if (ordered) {
@@ -174,25 +200,29 @@ Network::checkNetworkAllocation(NodeID id, bool ordered,
 
 
 void
-Network::setToNetQueue(NodeID id, bool ordered, int network_num,
+Network::setToNetQueue(NodeID global_id, bool ordered, int network_num,
                                  std::string vnet_type, MessageBuffer *b)
 {
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    while (m_toNetQueues[id].size() <= network_num) {
-        m_toNetQueues[id].push_back(nullptr);
+    NodeID local_id = getLocalNodeID(global_id);
+    checkNetworkAllocation(local_id, ordered, network_num, vnet_type);
+
+    while (m_toNetQueues[local_id].size() <= network_num) {
+        m_toNetQueues[local_id].push_back(nullptr);
     }
-    m_toNetQueues[id][network_num] = b;
+    m_toNetQueues[local_id][network_num] = b;
 }
 
 void
-Network::setFromNetQueue(NodeID id, bool ordered, int network_num,
+Network::setFromNetQueue(NodeID global_id, bool ordered, int network_num,
                                    std::string vnet_type, MessageBuffer *b)
 {
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    while (m_fromNetQueues[id].size() <= network_num) {
-        m_fromNetQueues[id].push_back(nullptr);
+    NodeID local_id = getLocalNodeID(global_id);
+    checkNetworkAllocation(local_id, ordered, network_num, vnet_type);
+
+    while (m_fromNetQueues[local_id].size() <= network_num) {
+        m_fromNetQueues[local_id].push_back(nullptr);
     }
-    m_fromNetQueues[id][network_num] = b;
+    m_fromNetQueues[local_id][network_num] = b;
 }
 
 NodeID
@@ -212,3 +242,10 @@ Network::addressToNodeID(Addr addr, MachineType mtype)
     }
     return MachineType_base_count(mtype);
 }
+
+NodeID
+Network::getLocalNodeID(NodeID global_id) const
+{
+    assert(globalToLocalMap.count(global_id));
+    return globalToLocalMap.at(global_id);
+}

diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh
index 606e6704bf3fb08755a1e5eb2eb7d146db8e2323..bba0c5e7f1f9307a4eb26e2bf349e1e3fb925653 100644 (file)
--- a/src/mem/ruby/network/Network.hh
+++ b/src/mem/ruby/network/Network.hh
@@ -90,13 +90,13 @@ class Network : public ClockedObject
     static uint32_t MessageSizeType_to_int(MessageSizeType size_type);
 
     // returns the queue requested for the given component
-    void setToNetQueue(NodeID id, bool ordered, int netNumber,
+    void setToNetQueue(NodeID global_id, bool ordered, int netNumber,
                                std::string vnet_type, MessageBuffer *b);
-    virtual void setFromNetQueue(NodeID id, bool ordered, int netNumber,
+    virtual void setFromNetQueue(NodeID global_id, bool ordered, int netNumber,
                                  std::string vnet_type, MessageBuffer *b);
 
-    virtual void checkNetworkAllocation(NodeID id, bool ordered,
-        int network_num, std::string vnet_type);
+    virtual void checkNetworkAllocation(NodeID local_id, bool ordered,
+                                       int network_num, std::string vnet_type);
 
     virtual void makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
                              const NetDest& routing_table_entry) = 0;
@@ -140,6 +140,8 @@ class Network : public ClockedObject
         return RubyDummyPort::instance();
     }
 
+    NodeID getLocalNodeID(NodeID global_id) const;
+
   protected:
     // Private copy constructor and assignment operator
     Network(const Network& obj);
@@ -182,6 +184,10 @@ class Network : public ClockedObject
         AddrRangeList ranges;
     };
     std::unordered_multimap<MachineType, AddrMapNode> addrMap;
+
+    // Global NodeID to local node map. If there are not multiple networks in
+    // the same RubySystem, this is a one-to-one mapping of global to local.
+    std::unordered_map<NodeID, NodeID> globalToLocalMap;
 };
 
 inline std::ostream&

diff --git a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
index 8f226b1de2ec506b22051c6eac8f32b6e61b997a..a88302b6c4b6952a8a5a5f992e93868b9ee04d9b 100644 (file)
--- a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
+++ b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
@@ -146,10 +146,11 @@ GarnetNetwork::init()
 */
 
 void
-GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
+GarnetNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link,
                             const NetDest& routing_table_entry)
 {
-    assert(src < m_nodes);
+    NodeID local_src = getLocalNodeID(global_src);
+    assert(local_src < m_nodes);
 
     GarnetExtLink* garnet_link = safe_cast<GarnetExtLink*>(link);
 
@@ -163,7 +164,7 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
 
     PortDirection dst_inport_dirn = "Local";
     m_routers[dest]->addInPort(dst_inport_dirn, net_link, credit_link);
-    m_nis[src]->addOutPort(net_link, credit_link, dest);
+    m_nis[local_src]->addOutPort(net_link, credit_link, dest);
 }
 
 /*
@@ -173,10 +174,12 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
 */
 
 void
-GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
-                             const NetDest& routing_table_entry)
+GarnetNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
+                              BasicLink* link,
+                              const NetDest& routing_table_entry)
 {
-    assert(dest < m_nodes);
+    NodeID local_dest = getLocalNodeID(global_dest);
+    assert(local_dest < m_nodes);
     assert(src < m_routers.size());
     assert(m_routers[src] != NULL);
 
@@ -194,7 +197,7 @@ GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
     m_routers[src]->addOutPort(src_outport_dirn, net_link,
                                routing_table_entry,
                                link->m_weight, credit_link);
-    m_nis[dest]->addInPort(net_link, credit_link);
+    m_nis[local_dest]->addInPort(net_link, credit_link);
 }
 
 /*
@@ -233,9 +236,11 @@ GarnetNetwork::getNumRouters()
 
 // Get ID of router connected to a NI.
 int
-GarnetNetwork::get_router_id(int ni)
+GarnetNetwork::get_router_id(int global_ni)
 {
-    return m_nis[ni]->get_router_id();
+    NodeID local_ni = getLocalNodeID(global_ni);
+
+    return m_nis[local_ni]->get_router_id();
 }
 
 void

diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc
index 84817e444b25ed2741852e810c137fc9ef12704f..d3b5515123384509dd8e242111ab396ab49f74ca 100644 (file)
--- a/src/mem/ruby/network/simple/SimpleNetwork.cc
+++ b/src/mem/ruby/network/simple/SimpleNetwork.cc
@@ -83,27 +83,30 @@ SimpleNetwork::init()
 
 // From a switch to an endpoint node
 void
-SimpleNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
-                           const NetDest& routing_table_entry)
+SimpleNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
+                              BasicLink* link,
+                              const NetDest& routing_table_entry)
 {
-    assert(dest < m_nodes);
+    NodeID local_dest = getLocalNodeID(global_dest);
+    assert(local_dest < m_nodes);
     assert(src < m_switches.size());
     assert(m_switches[src] != NULL);
 
     SimpleExtLink *simple_link = safe_cast<SimpleExtLink*>(link);
 
-    m_switches[src]->addOutPort(m_fromNetQueues[dest], routing_table_entry,
-                                simple_link->m_latency,
+    m_switches[src]->addOutPort(m_fromNetQueues[local_dest],
+                                routing_table_entry, simple_link->m_latency,
                                 simple_link->m_bw_multiplier);
 }
 
 // From an endpoint node to a switch
 void
-SimpleNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
+SimpleNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link,
                           const NetDest& routing_table_entry)
 {
-    assert(src < m_nodes);
-    m_switches[dest]->addInPort(m_toNetQueues[src]);
+    NodeID local_src = getLocalNodeID(global_src);
+    assert(local_src < m_nodes);
+    m_switches[dest]->addInPort(m_toNetQueues[local_src]);
 }
 
 // From a switch to a switch

diff --git a/src/mem/ruby/system/RubySystem.cc b/src/mem/ruby/system/RubySystem.cc
index 2aeff8caebd11cae5f36a148ac533361b7c8e4f1..c9da20656bd88d709519b2dc553e38feab0e4cf4 100644 (file)
--- a/src/mem/ruby/system/RubySystem.cc
+++ b/src/mem/ruby/system/RubySystem.cc
@@ -94,7 +94,7 @@ RubySystem::RubySystem(const Params *p)
 void
 RubySystem::registerNetwork(Network* network_ptr)
 {
-    m_network = network_ptr;
+    m_networks.emplace_back(network_ptr);
 }
 
 void
@@ -108,7 +108,6 @@ RubySystem::registerAbstractController(AbstractController* cntrl)
 
 RubySystem::~RubySystem()
 {
-    delete m_network;
     delete m_profiler;
 }
 
@@ -407,7 +406,9 @@ void
 RubySystem::resetStats()
 {
     m_start_cycle = curCycle();
-    m_network->resetStats();
+    for (auto& network : m_networks) {
+        network->resetStats();
+    }
 }
 
 bool
@@ -512,8 +513,10 @@ RubySystem::functionalRead(PacketPtr pkt)
         DPRINTF(RubySystem, "Network functionalRead lookup "
                             "(num_maybe_stale=%d, num_busy = %d)\n",
                 num_maybe_stale, num_busy);
-        if (m_network->functionalRead(pkt))
-            return true;
+        for (auto& network : m_networks) {
+            if (network->functionalRead(pkt))
+                return true;
+        }
     }
 
     return false;
@@ -558,7 +561,9 @@ RubySystem::functionalWrite(PacketPtr pkt)
         }
     }
 
-    num_functional_writes += m_network->functionalWrite(pkt);
+    for (auto& network : m_networks) {
+        num_functional_writes += network->functionalWrite(pkt);
+    }
     DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes);
 
     return true;

diff --git a/src/mem/ruby/system/RubySystem.hh b/src/mem/ruby/system/RubySystem.hh
index 5d10991aaea1f24bf58506aa68f1ebcbb306527f..240707285079be161dd0518b417f21ea071ced69 100644 (file)
--- a/src/mem/ruby/system/RubySystem.hh
+++ b/src/mem/ruby/system/RubySystem.hh
@@ -130,7 +130,8 @@ class RubySystem : public ClockedObject
     SimpleMemory *m_phys_mem;
     const bool m_access_backing_store;
 
-    Network* m_network;
+    //std::vector<Network *> m_networks;
+    std::vector<std::unique_ptr<Network>> m_networks;
     std::vector<AbstractController *> m_abs_cntrl_vec;
     Cycles m_start_cycle;