src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc

   1 /*
   2  * Copyright (c) 2008 Princeton University
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met: redistributions of source code must retain the above copyright
   8  * notice, this list of conditions and the following disclaimer;
   9  * redistributions in binary form must reproduce the above copyright
  10  * notice, this list of conditions and the following disclaimer in the
  11  * documentation and/or other materials provided with the distribution;
  12  * neither the name of the copyright holders nor the names of its
  13  * contributors may be used to endorse or promote products derived from
  14  * this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * Authors: Niket Agarwal
  29  */
  30
  31 #include <cassert>
  32 #include <cmath>
  33
  34 #include "base/stl_helpers.hh"
  35 #include "mem/ruby/buffers/MessageBuffer.hh"
  36 #include "mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh"
  37 #include "mem/ruby/network/garnet/fixed-pipeline/flitBuffer_d.hh"
  38 #include "mem/ruby/slicc_interface/NetworkMessage.hh"
  39
  40 using namespace std;
  41 using m5::stl_helpers::deletePointers;
  42
  43 NetworkInterface_d::NetworkInterface_d(int id, int virtual_networks,
  44                                        GarnetNetwork_d *network_ptr)
  45 {
  46     m_id = id;
  47     m_net_ptr = network_ptr;
  48     m_virtual_networks  = virtual_networks;
  49     m_vc_per_vnet = m_net_ptr->getVCsPerClass();
  50     m_num_vcs = m_vc_per_vnet*m_virtual_networks;
  51
  52     m_vc_round_robin = 0;
  53     m_ni_buffers.resize(m_num_vcs);
  54     m_ni_enqueue_time.resize(m_num_vcs);
  55     inNode_ptr.resize(m_virtual_networks);
  56     outNode_ptr.resize(m_virtual_networks);
  57     creditQueue = new flitBuffer_d();
  58
  59     // instantiating the NI flit buffers
  60     for (int i = 0; i < m_num_vcs; i++) {
  61         m_ni_buffers[i] = new flitBuffer_d();
  62         m_ni_enqueue_time[i] = INFINITE_;
  63     }
  64     m_vc_allocator.resize(m_virtual_networks); // 1 allocator per vnet
  65     for (int i = 0; i < m_virtual_networks; i++) {
  66         m_vc_allocator[i] = 0;
  67     }
  68
  69     for (int i = 0; i < m_num_vcs; i++) {
  70         m_out_vc_state.push_back(new OutVcState_d(i, m_net_ptr));
  71         m_out_vc_state[i]->setState(IDLE_, g_eventQueue_ptr->getTime());
  72     }
  73 }
  74
  75 NetworkInterface_d::~NetworkInterface_d()
  76 {
  77     deletePointers(m_out_vc_state);
  78     deletePointers(m_ni_buffers);
  79     delete creditQueue;
  80     delete outSrcQueue;
  81 }
  82
  83 void
  84 NetworkInterface_d::addInPort(NetworkLink_d *in_link,
  85                               CreditLink_d *credit_link)
  86 {
  87     inNetLink = in_link;
  88     in_link->setLinkConsumer(this);
  89     m_ni_credit_link = credit_link;
  90     credit_link->setSourceQueue(creditQueue);
  91 }
  92
  93 void
  94 NetworkInterface_d::addOutPort(NetworkLink_d *out_link,
  95                                CreditLink_d *credit_link)
  96 {
  97     m_credit_link = credit_link;
  98     credit_link->setLinkConsumer(this);
  99
 100     outNetLink = out_link;
 101     outSrcQueue = new flitBuffer_d();
 102     out_link->setSourceQueue(outSrcQueue);
 103 }
 104
 105 void
 106 NetworkInterface_d::addNode(vector<MessageBuffer *>& in,
 107                             vector<MessageBuffer *>& out)
 108 {
 109     assert(in.size() == m_virtual_networks);
 110     inNode_ptr = in;
 111     outNode_ptr = out;
 112     for (int j = 0; j < m_virtual_networks; j++) {
 113
 114         // the protocol injects messages into the NI
 115         inNode_ptr[j]->setConsumer(this);
 116     }
 117 }
 118
 119 bool
 120 NetworkInterface_d::flitisizeMessage(MsgPtr msg_ptr, int vnet)
 121 {
 122     NetworkMessage *net_msg_ptr = safe_cast<NetworkMessage *>(msg_ptr.get());
 123     NetDest net_msg_dest = net_msg_ptr->getInternalDestination();
 124
 125     // gets all the destinations associated with this message.
 126     vector<NodeID> dest_nodes = net_msg_dest.getAllDest();
 127
 128     // Number of flits is dependent on the link bandwidth available.
 129     // This is expressed in terms of bytes/cycle or the flit size
 130     int num_flits = (int) ceil((double) m_net_ptr->MessageSizeType_to_int(
 131         net_msg_ptr->getMessageSize())/m_net_ptr->getFlitSize() );
 132
 133     // loop to convert all multicast messages into unicast messages
 134     for (int ctr = 0; ctr < dest_nodes.size(); ctr++) {
 135
 136         // this will return a free output virtual channel
 137         int vc = calculateVC(vnet);
 138
 139         if (vc == -1) {
 140             return false ;
 141         }
 142         MsgPtr new_msg_ptr = msg_ptr->clone();
 143         NodeID destID = dest_nodes[ctr];
 144
 145         NetworkMessage *new_net_msg_ptr =
 146             safe_cast<NetworkMessage *>(new_msg_ptr.get());
 147         if (dest_nodes.size() > 1) {
 148             NetDest personal_dest;
 149             for (int m = 0; m < (int) MachineType_NUM; m++) {
 150                 if ((destID >= MachineType_base_number((MachineType) m)) &&
 151                     destID < MachineType_base_number((MachineType) (m+1))) {
 152                     // calculating the NetDest associated with this destID
 153                     personal_dest.clear();
 154                     personal_dest.add((MachineID) {(MachineType) m, (destID -
 155                         MachineType_base_number((MachineType) m))});
 156                     new_net_msg_ptr->getInternalDestination() = personal_dest;
 157                     break;
 158                 }
 159             }
 160             net_msg_dest.removeNetDest(personal_dest);
 161             // removing the destination from the original message to reflect
 162             // that a message with this particular destination has been
 163             // flitisized and an output vc is acquired
 164             net_msg_ptr->getInternalDestination().removeNetDest(personal_dest);
 165         }
 166         if (num_flits > 1) { // data packet
 167             // defining ctrl vnet to be 1-flit packets
 168             // and data vnet to be > 1 flit packets
 169             m_net_ptr->set_vnet_type(vc, DATA_VNET_);
 170             m_out_vc_state[vc]->set_credit_count();
 171         }
 172
 173         for (int i = 0; i < num_flits; i++) {
 174             m_net_ptr->increment_injected_flits();
 175             flit_d *fl = new flit_d(i, vc, vnet, num_flits, new_msg_ptr);
 176             fl->set_delay(g_eventQueue_ptr->getTime() - msg_ptr->getTime());
 177             m_ni_buffers[vc]->insert(fl);
 178         }
 179         m_ni_enqueue_time[vc] = g_eventQueue_ptr->getTime();
 180         m_out_vc_state[vc]->setState(ACTIVE_, g_eventQueue_ptr->getTime());
 181     }
 182     return true ;
 183 }
 184
 185 // Looking for a free output vc
 186 int
 187 NetworkInterface_d::calculateVC(int vnet)
 188 {
 189         for (int i = 0; i < m_vc_per_vnet; i++) {
 190                 int delta = m_vc_allocator[vnet];
 191                 m_vc_allocator[vnet]++;
 192                 if(m_vc_allocator[vnet] == m_vc_per_vnet)
 193                         m_vc_allocator[vnet] = 0;
 194
 195                 if (m_out_vc_state[(vnet*m_vc_per_vnet) + delta]->isInState(
 196                     IDLE_, g_eventQueue_ptr->getTime())) {
 197                         return ((vnet*m_vc_per_vnet) + delta);
 198                 }
 199         }
 200         return -1;
 201 }
 202
 203 /*
 204  * The NI wakeup checks whether there are any ready messages in the protocol
 205  * buffer. If yes, it picks that up, flitisizes it into a number of flits and
 206  * puts it into an output buffer and schedules the output link. On a wakeup
 207  * it also checks whether there are flits in the input link. If yes, it picks
 208  * them up and if the flit is a tail, the NI inserts the corresponding message
 209  * into the protocol buffer. It also checks for credits being sent by the
 210  * downstream router.
 211  */
 212
 213 void
 214 NetworkInterface_d::wakeup()
 215 {
 216     DPRINTF(RubyNetwork, "m_id: %d woke up at time: %lld",
 217             m_id, g_eventQueue_ptr->getTime());
 218
 219     MsgPtr msg_ptr;
 220
 221     // Checking for messages coming from the protocol
 222     // can pick up a message/cycle for each virtual net
 223     for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
 224         while (inNode_ptr[vnet]->isReady()) { // Is there a message waiting
 225             msg_ptr = inNode_ptr[vnet]->peekMsgPtr();
 226             if (flitisizeMessage(msg_ptr, vnet)) {
 227                 inNode_ptr[vnet]->pop();
 228             } else {
 229                 break;
 230             }
 231         }
 232     }
 233
 234     scheduleOutputLink();
 235     checkReschedule();
 236
 237     /*********** Picking messages destined for this NI **********/
 238
 239     if (inNetLink->isReady()) {
 240         flit_d *t_flit = inNetLink->consumeLink();
 241         bool free_signal = false;
 242         if (t_flit->get_type() == TAIL_ || t_flit->get_type() == HEAD_TAIL_) {
 243             free_signal = true;
 244
 245             outNode_ptr[t_flit->get_vnet()]->enqueue(
 246                 t_flit->get_msg_ptr(), 1);
 247         }
 248         // Simply send a credit back since we are not buffering
 249         // this flit in the NI
 250         flit_d *credit_flit = new flit_d(t_flit->get_vc(), free_signal);
 251         creditQueue->insert(credit_flit);
 252         g_eventQueue_ptr->scheduleEvent(m_ni_credit_link, 1);
 253
 254         m_net_ptr->increment_received_flits();
 255         int network_delay = g_eventQueue_ptr->getTime() -
 256                             t_flit->get_enqueue_time();
 257         int queueing_delay = t_flit->get_delay();
 258         m_net_ptr->increment_network_latency(network_delay);
 259         m_net_ptr->increment_queueing_latency(queueing_delay);
 260         delete t_flit;
 261     }
 262
 263     /****************** Checking for credit link *******/
 264
 265     if (m_credit_link->isReady()) {
 266         flit_d *t_flit = m_credit_link->consumeLink();
 267         m_out_vc_state[t_flit->get_vc()]->increment_credit();
 268         if (t_flit->is_free_signal()) {
 269             m_out_vc_state[t_flit->get_vc()]->setState(IDLE_,
 270                 g_eventQueue_ptr->getTime());
 271         }
 272         delete t_flit;
 273     }
 274 }
 275
 276 /** This function looks at the NI buffers
 277  *  if some buffer has flits which are ready to traverse the link in the next
 278  *  cycle, and the downstream output vc associated with this flit has buffers
 279  *  left, the link is scheduled for the next cycle
 280  */
 281
 282 void
 283 NetworkInterface_d::scheduleOutputLink()
 284 {
 285     int vc = m_vc_round_robin;
 286     m_vc_round_robin++;
 287     if (m_vc_round_robin == m_num_vcs)
 288         m_vc_round_robin = 0;
 289
 290     for (int i = 0; i < m_num_vcs; i++) {
 291         vc++;
 292         if (vc == m_num_vcs)
 293             vc = 0;
 294
 295         // model buffer backpressure
 296         if (m_ni_buffers[vc]->isReady() && m_out_vc_state[vc]->has_credits()) {
 297             bool is_candidate_vc = true;
 298             int t_vnet = get_vnet(vc);
 299             int vc_base = t_vnet * m_vc_per_vnet;
 300
 301             if (m_net_ptr->isVNetOrdered(t_vnet)) {
 302                 for (int vc_offset = 0; vc_offset < m_vc_per_vnet;
 303                      vc_offset++) {
 304                     int t_vc = vc_base + vc_offset;
 305                     if (m_ni_buffers[t_vc]->isReady()) {
 306                         if (m_ni_enqueue_time[t_vc] < m_ni_enqueue_time[vc]) {
 307                             is_candidate_vc = false;
 308                             break;
 309                         }
 310                     }
 311                 }
 312             }
 313             if (!is_candidate_vc)
 314                 continue;
 315
 316             m_out_vc_state[vc]->decrement_credit();
 317             // Just removing the flit
 318             flit_d *t_flit = m_ni_buffers[vc]->getTopFlit();
 319             t_flit->set_time(g_eventQueue_ptr->getTime() + 1);
 320             outSrcQueue->insert(t_flit);
 321             // schedule the out link
 322             g_eventQueue_ptr->scheduleEvent(outNetLink, 1);
 323
 324             if (t_flit->get_type() == TAIL_ ||
 325                t_flit->get_type() == HEAD_TAIL_) {
 326                 m_ni_enqueue_time[vc] = INFINITE_;
 327             }
 328             return;
 329         }
 330     }
 331 }
 332
 333 int
 334 NetworkInterface_d::get_vnet(int vc)
 335 {
 336     for (int i = 0; i < m_net_ptr->getNumberOfVirtualNetworks(); i++) {
 337         if (vc >= (i*m_vc_per_vnet) && vc < ((i+1)*m_vc_per_vnet)) {
 338             return i;
 339         }
 340     }
 341     fatal("Could not determine vc");
 342 }
 343
 344 void
 345 NetworkInterface_d::checkReschedule()
 346 {
 347     for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
 348         if (inNode_ptr[vnet]->isReady()) { // Is there a message waiting
 349             g_eventQueue_ptr->scheduleEvent(this, 1);
 350             return;
 351         }
 352     }
 353     for (int vc = 0; vc < m_num_vcs; vc++) {
 354         if (m_ni_buffers[vc]->isReadyForNext()) {
 355             g_eventQueue_ptr->scheduleEvent(this, 1);
 356             return;
 357         }
 358     }
 359 }
 360
 361 void
 362 NetworkInterface_d::printConfig(std::ostream& out) const
 363 {
 364     out << "[Network Interface " << m_id << "] - ";
 365     out << "[inLink " << inNetLink->get_id() << "] - ";
 366     out << "[outLink " << outNetLink->get_id() << "]" << std::endl;
 367 }
 368
 369 void
 370 NetworkInterface_d::print(std::ostream& out) const
 371 {
 372     out << "[Network Interface]";
 373 }