src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc

   1 /*
   2  * Copyright (c) 2008 Princeton University
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met: redistributions of source code must retain the above copyright
   8  * notice, this list of conditions and the following disclaimer;
   9  * redistributions in binary form must reproduce the above copyright
  10  * notice, this list of conditions and the following disclaimer in the
  11  * documentation and/or other materials provided with the distribution;
  12  * neither the name of the copyright holders nor the names of its
  13  * contributors may be used to endorse or promote products derived from
  14  * this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * Authors: Niket Agarwal
  29  */
  30
  31 #include <cassert>
  32 #include <cmath>
  33
  34 #include "base/cast.hh"
  35 #include "base/stl_helpers.hh"
  36 #include "debug/RubyNetwork.hh"
  37 #include "mem/ruby/buffers/MessageBuffer.hh"
  38 #include "mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh"
  39 #include "mem/ruby/network/garnet/fixed-pipeline/flitBuffer_d.hh"
  40 #include "mem/ruby/slicc_interface/NetworkMessage.hh"
  41
  42 using namespace std;
  43 using m5::stl_helpers::deletePointers;
  44
  45 NetworkInterface_d::NetworkInterface_d(int id, int virtual_networks,
  46                                        GarnetNetwork_d *network_ptr)
  47     : Consumer(network_ptr)
  48 {
  49     m_id = id;
  50     m_net_ptr = network_ptr;
  51     m_virtual_networks  = virtual_networks;
  52     m_vc_per_vnet = m_net_ptr->getVCsPerVnet();
  53     m_num_vcs = m_vc_per_vnet*m_virtual_networks;
  54
  55     m_vc_round_robin = 0;
  56     m_ni_buffers.resize(m_num_vcs);
  57     m_ni_enqueue_time.resize(m_num_vcs);
  58     inNode_ptr.resize(m_virtual_networks);
  59     outNode_ptr.resize(m_virtual_networks);
  60     creditQueue = new flitBuffer_d();
  61
  62     // instantiating the NI flit buffers
  63     for (int i = 0; i < m_num_vcs; i++) {
  64         m_ni_buffers[i] = new flitBuffer_d();
  65         m_ni_enqueue_time[i] = INFINITE_;
  66     }
  67     m_vc_allocator.resize(m_virtual_networks); // 1 allocator per vnet
  68     for (int i = 0; i < m_virtual_networks; i++) {
  69         m_vc_allocator[i] = 0;
  70     }
  71
  72     for (int i = 0; i < m_num_vcs; i++) {
  73         m_out_vc_state.push_back(new OutVcState_d(i, m_net_ptr));
  74         m_out_vc_state[i]->setState(IDLE_, m_net_ptr->curCycle());
  75     }
  76 }
  77
  78 NetworkInterface_d::~NetworkInterface_d()
  79 {
  80     deletePointers(m_out_vc_state);
  81     deletePointers(m_ni_buffers);
  82     delete creditQueue;
  83     delete outSrcQueue;
  84 }
  85
  86 void
  87 NetworkInterface_d::addInPort(NetworkLink_d *in_link,
  88                               CreditLink_d *credit_link)
  89 {
  90     inNetLink = in_link;
  91     in_link->setLinkConsumer(this);
  92     m_ni_credit_link = credit_link;
  93     credit_link->setSourceQueue(creditQueue);
  94 }
  95
  96 void
  97 NetworkInterface_d::addOutPort(NetworkLink_d *out_link,
  98                                CreditLink_d *credit_link)
  99 {
 100     m_credit_link = credit_link;
 101     credit_link->setLinkConsumer(this);
 102
 103     outNetLink = out_link;
 104     outSrcQueue = new flitBuffer_d();
 105     out_link->setSourceQueue(outSrcQueue);
 106 }
 107
 108 void
 109 NetworkInterface_d::addNode(vector<MessageBuffer *>& in,
 110                             vector<MessageBuffer *>& out)
 111 {
 112     assert(in.size() == m_virtual_networks);
 113     inNode_ptr = in;
 114     outNode_ptr = out;
 115     for (int j = 0; j < m_virtual_networks; j++) {
 116
 117         // the protocol injects messages into the NI
 118         inNode_ptr[j]->setConsumer(this);
 119         inNode_ptr[j]->setReceiver(m_net_ptr);
 120
 121         outNode_ptr[j]->setSender(m_net_ptr);
 122     }
 123 }
 124
 125 bool
 126 NetworkInterface_d::flitisizeMessage(MsgPtr msg_ptr, int vnet)
 127 {
 128     NetworkMessage *net_msg_ptr = safe_cast<NetworkMessage *>(msg_ptr.get());
 129     NetDest net_msg_dest = net_msg_ptr->getInternalDestination();
 130
 131     // gets all the destinations associated with this message.
 132     vector<NodeID> dest_nodes = net_msg_dest.getAllDest();
 133
 134     // Number of flits is dependent on the link bandwidth available.
 135     // This is expressed in terms of bytes/cycle or the flit size
 136     int num_flits = (int) ceil((double) m_net_ptr->MessageSizeType_to_int(
 137         net_msg_ptr->getMessageSize())/m_net_ptr->getNiFlitSize());
 138
 139     // loop to convert all multicast messages into unicast messages
 140     for (int ctr = 0; ctr < dest_nodes.size(); ctr++) {
 141
 142         // this will return a free output virtual channel
 143         int vc = calculateVC(vnet);
 144
 145         if (vc == -1) {
 146             return false ;
 147         }
 148         MsgPtr new_msg_ptr = msg_ptr->clone();
 149         NodeID destID = dest_nodes[ctr];
 150
 151         NetworkMessage *new_net_msg_ptr =
 152             safe_cast<NetworkMessage *>(new_msg_ptr.get());
 153         if (dest_nodes.size() > 1) {
 154             NetDest personal_dest;
 155             for (int m = 0; m < (int) MachineType_NUM; m++) {
 156                 if ((destID >= MachineType_base_number((MachineType) m)) &&
 157                     destID < MachineType_base_number((MachineType) (m+1))) {
 158                     // calculating the NetDest associated with this destID
 159                     personal_dest.clear();
 160                     personal_dest.add((MachineID) {(MachineType) m, (destID -
 161                         MachineType_base_number((MachineType) m))});
 162                     new_net_msg_ptr->getInternalDestination() = personal_dest;
 163                     break;
 164                 }
 165             }
 166             net_msg_dest.removeNetDest(personal_dest);
 167             // removing the destination from the original message to reflect
 168             // that a message with this particular destination has been
 169             // flitisized and an output vc is acquired
 170             net_msg_ptr->getInternalDestination().removeNetDest(personal_dest);
 171         }
 172
 173         for (int i = 0; i < num_flits; i++) {
 174             m_net_ptr->increment_injected_flits(vnet);
 175             flit_d *fl = new flit_d(i, vc, vnet, num_flits, new_msg_ptr,
 176                 m_net_ptr->curCycle());
 177
 178             fl->set_delay(m_net_ptr->curCycle() -
 179                           m_net_ptr->ticksToCycles(msg_ptr->getTime()));
 180             m_ni_buffers[vc]->insert(fl);
 181         }
 182
 183         m_ni_enqueue_time[vc] = m_net_ptr->curCycle();
 184         m_out_vc_state[vc]->setState(ACTIVE_, m_net_ptr->curCycle());
 185     }
 186     return true ;
 187 }
 188
 189 // Looking for a free output vc
 190 int
 191 NetworkInterface_d::calculateVC(int vnet)
 192 {
 193         for (int i = 0; i < m_vc_per_vnet; i++) {
 194                 int delta = m_vc_allocator[vnet];
 195                 m_vc_allocator[vnet]++;
 196                 if(m_vc_allocator[vnet] == m_vc_per_vnet)
 197                         m_vc_allocator[vnet] = 0;
 198
 199                 if (m_out_vc_state[(vnet*m_vc_per_vnet) + delta]->isInState(
 200                     IDLE_, m_net_ptr->curCycle())) {
 201                         return ((vnet*m_vc_per_vnet) + delta);
 202                 }
 203         }
 204         return -1;
 205 }
 206
 207 /*
 208  * The NI wakeup checks whether there are any ready messages in the protocol
 209  * buffer. If yes, it picks that up, flitisizes it into a number of flits and
 210  * puts it into an output buffer and schedules the output link. On a wakeup
 211  * it also checks whether there are flits in the input link. If yes, it picks
 212  * them up and if the flit is a tail, the NI inserts the corresponding message
 213  * into the protocol buffer. It also checks for credits being sent by the
 214  * downstream router.
 215  */
 216
 217 void
 218 NetworkInterface_d::wakeup()
 219 {
 220     DPRINTF(RubyNetwork, "m_id: %d woke up at time: %lld",
 221             m_id, m_net_ptr->curCycle());
 222
 223     MsgPtr msg_ptr;
 224
 225     // Checking for messages coming from the protocol
 226     // can pick up a message/cycle for each virtual net
 227     for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
 228         while (inNode_ptr[vnet]->isReady()) { // Is there a message waiting
 229             msg_ptr = inNode_ptr[vnet]->peekMsgPtr();
 230             if (flitisizeMessage(msg_ptr, vnet)) {
 231                 inNode_ptr[vnet]->pop();
 232             } else {
 233                 break;
 234             }
 235         }
 236     }
 237
 238     scheduleOutputLink();
 239     checkReschedule();
 240
 241     /*********** Picking messages destined for this NI **********/
 242
 243     if (inNetLink->isReady(m_net_ptr->curCycle())) {
 244         flit_d *t_flit = inNetLink->consumeLink();
 245         bool free_signal = false;
 246         if (t_flit->get_type() == TAIL_ || t_flit->get_type() == HEAD_TAIL_) {
 247             free_signal = true;
 248
 249             outNode_ptr[t_flit->get_vnet()]->enqueue(
 250                 t_flit->get_msg_ptr(), Cycles(1));
 251         }
 252         // Simply send a credit back since we are not buffering
 253         // this flit in the NI
 254         flit_d *credit_flit = new flit_d(t_flit->get_vc(), free_signal,
 255                                          m_net_ptr->curCycle());
 256         creditQueue->insert(credit_flit);
 257         m_ni_credit_link->scheduleEvent(Cycles(1));
 258
 259         int vnet = t_flit->get_vnet();
 260         m_net_ptr->increment_received_flits(vnet);
 261         Cycles network_delay = m_net_ptr->curCycle() -
 262                                t_flit->get_enqueue_time();
 263         Cycles queueing_delay = t_flit->get_delay();
 264
 265         m_net_ptr->increment_network_latency(network_delay, vnet);
 266         m_net_ptr->increment_queueing_latency(queueing_delay, vnet);
 267         delete t_flit;
 268     }
 269
 270     /****************** Checking for credit link *******/
 271
 272     if (m_credit_link->isReady(m_net_ptr->curCycle())) {
 273         flit_d *t_flit = m_credit_link->consumeLink();
 274         m_out_vc_state[t_flit->get_vc()]->increment_credit();
 275         if (t_flit->is_free_signal()) {
 276             m_out_vc_state[t_flit->get_vc()]->setState(IDLE_,
 277                 m_net_ptr->curCycle());
 278         }
 279         delete t_flit;
 280     }
 281 }
 282
 283 /** This function looks at the NI buffers
 284  *  if some buffer has flits which are ready to traverse the link in the next
 285  *  cycle, and the downstream output vc associated with this flit has buffers
 286  *  left, the link is scheduled for the next cycle
 287  */
 288
 289 void
 290 NetworkInterface_d::scheduleOutputLink()
 291 {
 292     int vc = m_vc_round_robin;
 293     m_vc_round_robin++;
 294     if (m_vc_round_robin == m_num_vcs)
 295         m_vc_round_robin = 0;
 296
 297     for (int i = 0; i < m_num_vcs; i++) {
 298         vc++;
 299         if (vc == m_num_vcs)
 300             vc = 0;
 301
 302         // model buffer backpressure
 303         if (m_ni_buffers[vc]->isReady(m_net_ptr->curCycle()) &&
 304             m_out_vc_state[vc]->has_credits()) {
 305
 306             bool is_candidate_vc = true;
 307             int t_vnet = get_vnet(vc);
 308             int vc_base = t_vnet * m_vc_per_vnet;
 309
 310             if (m_net_ptr->isVNetOrdered(t_vnet)) {
 311                 for (int vc_offset = 0; vc_offset < m_vc_per_vnet;
 312                      vc_offset++) {
 313                     int t_vc = vc_base + vc_offset;
 314                     if (m_ni_buffers[t_vc]->isReady(m_net_ptr->curCycle())) {
 315                         if (m_ni_enqueue_time[t_vc] < m_ni_enqueue_time[vc]) {
 316                             is_candidate_vc = false;
 317                             break;
 318                         }
 319                     }
 320                 }
 321             }
 322             if (!is_candidate_vc)
 323                 continue;
 324
 325             m_out_vc_state[vc]->decrement_credit();
 326             // Just removing the flit
 327             flit_d *t_flit = m_ni_buffers[vc]->getTopFlit();
 328             t_flit->set_time(m_net_ptr->curCycle() + Cycles(1));
 329             outSrcQueue->insert(t_flit);
 330             // schedule the out link
 331             outNetLink->scheduleEvent(Cycles(1));
 332
 333             if (t_flit->get_type() == TAIL_ ||
 334                t_flit->get_type() == HEAD_TAIL_) {
 335                 m_ni_enqueue_time[vc] = INFINITE_;
 336             }
 337             return;
 338         }
 339     }
 340 }
 341
 342 int
 343 NetworkInterface_d::get_vnet(int vc)
 344 {
 345     for (int i = 0; i < m_net_ptr->getNumberOfVirtualNetworks(); i++) {
 346         if (vc >= (i*m_vc_per_vnet) && vc < ((i+1)*m_vc_per_vnet)) {
 347             return i;
 348         }
 349     }
 350     fatal("Could not determine vc");
 351 }
 352
 353 void
 354 NetworkInterface_d::checkReschedule()
 355 {
 356     for (int vnet = 0; vnet < m_virtual_networks; vnet++) {
 357         if (inNode_ptr[vnet]->isReady()) { // Is there a message waiting
 358             scheduleEvent(Cycles(1));
 359             return;
 360         }
 361     }
 362     for (int vc = 0; vc < m_num_vcs; vc++) {
 363         if (m_ni_buffers[vc]->isReadyForNext(m_net_ptr->curCycle())) {
 364             scheduleEvent(Cycles(1));
 365             return;
 366         }
 367     }
 368 }
 369
 370 void
 371 NetworkInterface_d::print(std::ostream& out) const
 372 {
 373     out << "[Network Interface]";
 374 }
 375
 376 uint32_t
 377 NetworkInterface_d::functionalWrite(Packet *pkt)
 378 {
 379     uint32_t num_functional_writes = 0;
 380     for (unsigned int i  = 0; i < m_num_vcs; ++i) {
 381         num_functional_writes += m_ni_buffers[i]->functionalWrite(pkt);
 382     }
 383
 384     num_functional_writes += outSrcQueue->functionalWrite(pkt);
 385     return num_functional_writes;
 386 }