From 2ccdfc547d5b58bdc859e4497658e972d7af5c45 Mon Sep 17 00:00:00 2001 From: Nilay Vaish Date: Mon, 15 Sep 2014 16:19:38 -0500 Subject: [PATCH] ruby: network: revert some of the changes from ad9c042dce54 The changeset ad9c042dce54 made changes to the structures under the network directory to use a map of buffers instead of vector of buffers. The reasoning was that not all vnets that are created are used and we needlessly allocate more buffers than required and then iterate over them while processing network messages. But the move to map resulted in a slow down which was pointed out by Andreas Hansson. This patch moves things back to using vector of message buffers. --- src/mem/ruby/network/Network.cc | 4 +- src/mem/ruby/network/Network.hh | 4 +- .../ruby/network/garnet/BaseGarnetNetwork.cc | 6 +++ .../fixed-pipeline/NetworkInterface_d.cc | 29 ++++++++------ .../fixed-pipeline/NetworkInterface_d.hh | 8 ++-- .../flexible-pipeline/NetworkInterface.cc | 29 ++++++++------ .../flexible-pipeline/NetworkInterface.hh | 8 ++-- src/mem/ruby/network/simple/PerfectSwitch.cc | 34 +++++++++------- src/mem/ruby/network/simple/PerfectSwitch.hh | 8 ++-- src/mem/ruby/network/simple/SimpleNetwork.cc | 9 ++++- src/mem/ruby/network/simple/Switch.cc | 18 +++++---- src/mem/ruby/network/simple/Switch.hh | 4 +- src/mem/ruby/network/simple/Throttle.cc | 39 +++++++++---------- src/mem/ruby/network/simple/Throttle.hh | 11 +++--- 14 files changed, 124 insertions(+), 87 deletions(-) diff --git a/src/mem/ruby/network/Network.cc b/src/mem/ruby/network/Network.cc index 2d9376b08..5f1fff195 100644 --- a/src/mem/ruby/network/Network.cc +++ b/src/mem/ruby/network/Network.cc @@ -85,11 +85,11 @@ Network::~Network() // Delete the Message Buffers for (auto& it : m_toNetQueues[node]) { - delete it.second; + delete it; } for (auto& it : m_fromNetQueues[node]) { - delete it.second; + delete it; } } diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh index d595ca285..6b6258b2b 100644 --- a/src/mem/ruby/network/Network.hh +++ b/src/mem/ruby/network/Network.hh @@ -112,8 +112,8 @@ class Network : public ClockedObject static uint32_t m_data_msg_size; // vector of queues from the components - std::vector > m_toNetQueues; - std::vector > m_fromNetQueues; + std::vector > m_toNetQueues; + std::vector > m_fromNetQueues; std::vector m_in_use; std::vector m_ordered; diff --git a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc index 2aeddad37..ed4679c63 100644 --- a/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc +++ b/src/mem/ruby/network/garnet/BaseGarnetNetwork.cc @@ -71,6 +71,9 @@ BaseGarnetNetwork::setToNetQueue(NodeID id, bool ordered, int network_num, string vnet_type, MessageBuffer *b) { checkNetworkAllocation(id, ordered, network_num, vnet_type); + while (m_toNetQueues[id].size() <= network_num) { + m_toNetQueues[id].push_back(nullptr); + } m_toNetQueues[id][network_num] = b; } @@ -79,6 +82,9 @@ BaseGarnetNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num, string vnet_type, MessageBuffer *b) { checkNetworkAllocation(id, ordered, network_num, vnet_type); + while (m_fromNetQueues[id].size() <= network_num) { + m_fromNetQueues[id].push_back(nullptr); + } m_fromNetQueues[id][network_num] = b; } diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc index 7384cc6a7..47ded231c 100644 --- a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc +++ b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc @@ -106,20 +106,23 @@ NetworkInterface_d::addOutPort(NetworkLink_d *out_link, } void -NetworkInterface_d::addNode(map& in, - map& out) +NetworkInterface_d::addNode(vector& in, + vector& out) { inNode_ptr = in; outNode_ptr = out; for (auto& it : in) { - // the protocol injects messages into the NI - it.second->setConsumer(this); - it.second->setReceiver(this); + if (it != nullptr) { + it->setConsumer(this); + it->setReceiver(this); + } } for (auto& it : out) { - it.second->setSender(this); + if (it != nullptr) { + it->setSender(this); + } } } @@ -223,9 +226,11 @@ NetworkInterface_d::wakeup() // Checking for messages coming from the protocol // can pick up a message/cycle for each virtual net - for (auto it = inNode_ptr.begin(); it != inNode_ptr.end(); ++it) { - int vnet = (*it).first; - MessageBuffer *b = (*it).second; + for (int vnet = 0; vnet < inNode_ptr.size(); ++vnet) { + MessageBuffer *b = inNode_ptr[vnet]; + if (b == nullptr) { + continue; + } while (b->isReady()) { // Is there a message waiting msg_ptr = b->peekMsgPtr(); @@ -355,9 +360,11 @@ void NetworkInterface_d::checkReschedule() { for (const auto& it : inNode_ptr) { - MessageBuffer *b = it.second; + if (it == nullptr) { + continue; + } - while (b->isReady()) { // Is there a message waiting + while (it->isReady()) { // Is there a message waiting scheduleEvent(Cycles(1)); return; } diff --git a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh index 2494d05d1..0cb928d82 100644 --- a/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh +++ b/src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh @@ -60,8 +60,8 @@ class NetworkInterface_d : public ClockedObject, public Consumer void addOutPort(NetworkLink_d *out_link, CreditLink_d *credit_link); void wakeup(); - void addNode(std::map &inNode, - std::map &outNode); + void addNode(std::vector &inNode, + std::vector &outNode); void print(std::ostream& out) const; int get_vnet(int vc); @@ -90,9 +90,9 @@ class NetworkInterface_d : public ClockedObject, public Consumer std::vector m_ni_enqueue_time; // The Message buffers that takes messages from the protocol - std::map inNode_ptr; + std::vector inNode_ptr; // The Message buffers that provides messages to the protocol - std::map outNode_ptr; + std::vector outNode_ptr; bool flitisizeMessage(MsgPtr msg_ptr, int vnet); int calculateVC(int vnet); diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc index 26d2423e8..32066f0e1 100644 --- a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc +++ b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.cc @@ -90,20 +90,23 @@ NetworkInterface::addOutPort(NetworkLink *out_link) } void -NetworkInterface::addNode(map& in, - map& out) +NetworkInterface::addNode(vector& in, + vector& out) { inNode_ptr = in; outNode_ptr = out; for (auto& it: in) { - // the protocol injects messages into the NI - it.second->setConsumer(this); - it.second->setReceiver(this); + if (it != nullptr) { + it->setConsumer(this); + it->setReceiver(this); + } } for (auto& it : out) { - it.second->setSender(this); + if (it != nullptr) { + it->setSender(this); + } } } @@ -242,9 +245,11 @@ NetworkInterface::wakeup() //Checking for messages coming from the protocol // can pick up a message/cycle for each virtual net - for (auto it = inNode_ptr.begin(); it != inNode_ptr.end(); ++it) { - int vnet = (*it).first; - MessageBuffer *b = (*it).second; + for (int vnet = 0; vnet < inNode_ptr.size(); ++vnet) { + MessageBuffer *b = inNode_ptr[vnet]; + if (b == nullptr) { + continue; + } while (b->isReady()) { // Is there a message waiting msg_ptr = b->peekMsgPtr(); @@ -326,9 +331,11 @@ void NetworkInterface::checkReschedule() { for (const auto& it : inNode_ptr) { - MessageBuffer *b = it.second; + if (it == nullptr) { + continue; + } - while (b->isReady()) { // Is there a message waiting + while (it->isReady()) { // Is there a message waiting scheduleEvent(Cycles(1)); return; } diff --git a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh index aa30bd758..03cdf3dc6 100644 --- a/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh +++ b/src/mem/ruby/network/garnet/flexible-pipeline/NetworkInterface.hh @@ -56,8 +56,8 @@ class NetworkInterface : public ClockedObject, public FlexibleConsumer void addInPort(NetworkLink *in_link); void addOutPort(NetworkLink *out_link); - void addNode(std::map &inNode, - std::map &outNode); + void addNode(std::vector &inNode, + std::vector &outNode); void wakeup(); void grant_vc(int out_port, int vc, Cycles grant_time); @@ -93,10 +93,10 @@ class NetworkInterface : public ClockedObject, public FlexibleConsumer std::vector m_ni_buffers; // The Message buffers that takes messages from the protocol - std::map inNode_ptr; + std::vector inNode_ptr; // The Message buffers that provides messages to the protocol - std::map outNode_ptr; + std::vector outNode_ptr; bool flitisizeMessage(MsgPtr msg_ptr, int vnet); int calculateVC(int vnet); diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc index caf07b3cf..86cafbe15 100644 --- a/src/mem/ruby/network/simple/PerfectSwitch.cc +++ b/src/mem/ruby/network/simple/PerfectSwitch.cc @@ -68,26 +68,30 @@ PerfectSwitch::init(SimpleNetwork *network_ptr) } void -PerfectSwitch::addInPort(const map& in) +PerfectSwitch::addInPort(const vector& in) { NodeID port = m_in.size(); m_in.push_back(in); - for (auto& it : in) { - it.second->setConsumer(this); + for (int i = 0; i < in.size(); ++i) { + if (in[i] != nullptr) { + in[i]->setConsumer(this); - string desc = csprintf("[Queue from port %s %s %s to PerfectSwitch]", - to_string(m_switch_id), to_string(port), to_string(it.first)); + string desc = + csprintf("[Queue from port %s %s %s to PerfectSwitch]", + to_string(m_switch_id), to_string(port), + to_string(i)); - it.second->setDescription(desc); - it.second->setIncomingLink(port); - it.second->setVnet(it.first); + in[i]->setDescription(desc); + in[i]->setIncomingLink(port); + in[i]->setVnet(i); + } } } void -PerfectSwitch::addOutPort(const map& out, - const NetDest& routing_table_entry) +PerfectSwitch::addOutPort(const vector& out, + const NetDest& routing_table_entry) { // Setup link order LinkOrder l; @@ -131,10 +135,14 @@ PerfectSwitch::operateVnet(int vnet) vector output_link_destinations; // Is there a message waiting? - auto it = m_in[incoming].find(vnet); - if (it == m_in[incoming].end()) + if (m_in[incoming].size() <= vnet) { continue; - MessageBuffer *buffer = (*it).second; + } + + MessageBuffer *buffer = m_in[incoming][vnet]; + if (buffer == nullptr) { + continue; + } while (buffer->isReady()) { DPRINTF(RubyNetwork, "incoming: %d\n", incoming); diff --git a/src/mem/ruby/network/simple/PerfectSwitch.hh b/src/mem/ruby/network/simple/PerfectSwitch.hh index 161430bd1..f55281d54 100644 --- a/src/mem/ruby/network/simple/PerfectSwitch.hh +++ b/src/mem/ruby/network/simple/PerfectSwitch.hh @@ -65,8 +65,8 @@ class PerfectSwitch : public Consumer { return csprintf("PerfectSwitch-%i", m_switch_id); } void init(SimpleNetwork *); - void addInPort(const std::map& in); - void addOutPort(const std::map& out, + void addInPort(const std::vector& in); + void addOutPort(const std::vector& out, const NetDest& routing_table_entry); int getInLinks() const { return m_in.size(); } @@ -89,8 +89,8 @@ class PerfectSwitch : public Consumer SwitchID m_switch_id; // vector of queues from the components - std::vector > m_in; - std::vector > m_out; + std::vector > m_in; + std::vector > m_out; std::vector m_routing_table; std::vector m_link_order; diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc index f51a0c891..074052cb9 100644 --- a/src/mem/ruby/network/simple/SimpleNetwork.cc +++ b/src/mem/ruby/network/simple/SimpleNetwork.cc @@ -117,7 +117,8 @@ SimpleNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link, const NetDest& routing_table_entry) { // Create a set of new MessageBuffers - std::map queues; + std::vector queues(m_virtual_networks); + for (int i = 0; i < m_virtual_networks; i++) { // allocate a buffer MessageBuffer* buffer_ptr = new MessageBuffer; @@ -158,6 +159,9 @@ SimpleNetwork::setToNetQueue(NodeID id, bool ordered, int network_num, std::string vnet_type, MessageBuffer *b) { checkNetworkAllocation(id, ordered, network_num); + while (m_toNetQueues[id].size() <= network_num) { + m_toNetQueues[id].push_back(nullptr); + } m_toNetQueues[id][network_num] = b; } @@ -166,6 +170,9 @@ SimpleNetwork::setFromNetQueue(NodeID id, bool ordered, int network_num, std::string vnet_type, MessageBuffer *b) { checkNetworkAllocation(id, ordered, network_num); + while (m_fromNetQueues[id].size() <= network_num) { + m_fromNetQueues[id].push_back(nullptr); + } m_fromNetQueues[id][network_num] = b; } diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc index e028de02a..416a222bb 100644 --- a/src/mem/ruby/network/simple/Switch.cc +++ b/src/mem/ruby/network/simple/Switch.cc @@ -64,17 +64,19 @@ Switch::init() } void -Switch::addInPort(const map& in) +Switch::addInPort(const vector& in) { m_perfect_switch->addInPort(in); for (auto& it : in) { - it.second->setReceiver(this); + if (it != nullptr) { + it->setReceiver(this); + } } } void -Switch::addOutPort(const map& out, +Switch::addOutPort(const vector& out, const NetDest& routing_table_entry, Cycles link_latency, int bw_multiplier) { @@ -87,10 +89,12 @@ Switch::addOutPort(const map& out, m_throttles.push_back(throttle_ptr); // Create one buffer per vnet (these are intermediaryQueues) - map intermediateBuffers; + vector intermediateBuffers; - for (auto& it : out) { - it.second->setSender(this); + for (int i = 0; i < out.size(); ++i) { + if (out[i] != nullptr) { + out[i]->setSender(this); + } MessageBuffer* buffer_ptr = new MessageBuffer; // Make these queues ordered @@ -99,7 +103,7 @@ Switch::addOutPort(const map& out, buffer_ptr->resize(m_network_ptr->getBufferSize()); } - intermediateBuffers[it.first] = buffer_ptr; + intermediateBuffers.push_back(buffer_ptr); m_buffers_to_free.push_back(buffer_ptr); buffer_ptr->setSender(this); diff --git a/src/mem/ruby/network/simple/Switch.hh b/src/mem/ruby/network/simple/Switch.hh index d4e5c5eba..3c11a5fa0 100644 --- a/src/mem/ruby/network/simple/Switch.hh +++ b/src/mem/ruby/network/simple/Switch.hh @@ -62,8 +62,8 @@ class Switch : public BasicRouter ~Switch(); void init(); - void addInPort(const std::map& in); - void addOutPort(const std::map& out, + void addInPort(const std::vector& in); + void addOutPort(const std::vector& out, const NetDest& routing_table_entry, Cycles link_latency, int bw_multiplier); diff --git a/src/mem/ruby/network/simple/Throttle.cc b/src/mem/ruby/network/simple/Throttle.cc index 91bad217b..2da810edb 100644 --- a/src/mem/ruby/network/simple/Throttle.cc +++ b/src/mem/ruby/network/simple/Throttle.cc @@ -69,6 +69,8 @@ Throttle::init(NodeID node, Cycles link_latency, int link_bandwidth_multiplier, int endpoint_bandwidth) { m_node = node; + m_vnets = 0; + assert(link_bandwidth_multiplier > 0); m_link_bandwidth_multiplier = link_bandwidth_multiplier; @@ -80,23 +82,19 @@ Throttle::init(NodeID node, Cycles link_latency, } void -Throttle::addLinks(const map& in_vec, - const map& out_vec) +Throttle::addLinks(const vector& in_vec, + const vector& out_vec) { assert(in_vec.size() == out_vec.size()); - for (auto& it : in_vec) { - int vnet = it.first; - - auto jt = out_vec.find(vnet); - assert(jt != out_vec.end()); + for (int vnet = 0; vnet < in_vec.size(); ++vnet) { + MessageBuffer *in_ptr = in_vec[vnet]; + MessageBuffer *out_ptr = out_vec[vnet]; - MessageBuffer *in_ptr = it.second; - MessageBuffer *out_ptr = (*jt).second; - - m_in[vnet] = in_ptr; - m_out[vnet] = out_ptr; - m_units_remaining[vnet] = 0; + m_vnets++; + m_units_remaining.push_back(0); + m_in.push_back(in_ptr); + m_out.push_back(out_ptr); // Set consumer and description in_ptr->setConsumer(this); @@ -110,8 +108,9 @@ void Throttle::operateVnet(int vnet, int &bw_remaining, bool &schedule_wakeup, MessageBuffer *in, MessageBuffer *out) { - assert(out != NULL); - assert(in != NULL); + if (out == nullptr || in == nullptr) { + return; + } assert(m_units_remaining[vnet] >= 0); while (bw_remaining > 0 && (in->isReady() || m_units_remaining[vnet] > 0) && @@ -178,16 +177,14 @@ Throttle::wakeup() } if (iteration_direction) { - for (auto& it : m_in) { - int vnet = it.first; + for (int vnet = 0; vnet < m_vnets; ++vnet) { operateVnet(vnet, bw_remaining, schedule_wakeup, - it.second, m_out[vnet]); + m_in[vnet], m_out[vnet]); } } else { - for (auto it = m_in.rbegin(); it != m_in.rend(); ++it) { - int vnet = (*it).first; + for (int vnet = m_vnets-1; vnet >= 0; --vnet) { operateVnet(vnet, bw_remaining, schedule_wakeup, - (*it).second, m_out[vnet]); + m_in[vnet], m_out[vnet]); } } diff --git a/src/mem/ruby/network/simple/Throttle.hh b/src/mem/ruby/network/simple/Throttle.hh index d978f14fd..797511702 100644 --- a/src/mem/ruby/network/simple/Throttle.hh +++ b/src/mem/ruby/network/simple/Throttle.hh @@ -62,8 +62,8 @@ class Throttle : public Consumer std::string name() { return csprintf("Throttle-%i", m_sID); } - void addLinks(const std::map& in_vec, - const std::map& out_vec); + void addLinks(const std::vector& in_vec, + const std::vector& out_vec); void wakeup(); // The average utilization (a fraction) since last clearStats() @@ -92,9 +92,10 @@ class Throttle : public Consumer Throttle(const Throttle& obj); Throttle& operator=(const Throttle& obj); - std::map m_in; - std::map m_out; - std::map m_units_remaining; + std::vector m_in; + std::vector m_out; + unsigned int m_vnets; + std::vector m_units_remaining; int m_sID; NodeID m_node; -- 2.30.2