src/mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.cc

   1 /*
   2  * Copyright (c) 2008 Princeton University
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met: redistributions of source code must retain the above copyright
   8  * notice, this list of conditions and the following disclaimer;
   9  * redistributions in binary form must reproduce the above copyright
  10  * notice, this list of conditions and the following disclaimer in the
  11  * documentation and/or other materials provided with the distribution;
  12  * neither the name of the copyright holders nor the names of its
  13  * contributors may be used to endorse or promote products derived from
  14  * this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * Authors: Niket Agarwal
  29  */
  30
  31 #include <cassert>
  32 #include <cmath>
  33
  34 #include "base/cast.hh"
  35 #include "base/stl_helpers.hh"
  36 #include "debug/RubyNetwork.hh"
  37 #include "mem/ruby/network/MessageBuffer.hh"
  38 #include "mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh"
  39 #include "mem/ruby/network/garnet/fixed-pipeline/flitBuffer_d.hh"
  40 #include "mem/ruby/slicc_interface/NetworkMessage.hh"
  41
  42 using namespace std;
  43 using m5::stl_helpers::deletePointers;
  44
  45 NetworkInterface_d::NetworkInterface_d(const Params *p)
  46     : ClockedObject(p), Consumer(this)
  47 {
  48     m_id = p->id;
  49     m_virtual_networks  = p->virt_nets;
  50     m_vc_per_vnet = p->vcs_per_vnet;
  51     m_num_vcs = m_vc_per_vnet*m_virtual_networks;
  52
  53     m_vc_round_robin = 0;
  54     m_ni_buffers.resize(m_num_vcs);
  55     m_ni_enqueue_time.resize(m_num_vcs);
  56     creditQueue = new flitBuffer_d();
  57
  58     // instantiating the NI flit buffers
  59     for (int i = 0; i < m_num_vcs; i++) {
  60         m_ni_buffers[i] = new flitBuffer_d();
  61         m_ni_enqueue_time[i] = Cycles(INFINITE_);
  62     }
  63
  64     m_vc_allocator.resize(m_virtual_networks); // 1 allocator per vnet
  65     for (int i = 0; i < m_virtual_networks; i++) {
  66         m_vc_allocator[i] = 0;
  67     }
  68 }
  69
  70 void
  71 NetworkInterface_d::init()
  72 {
  73     for (int i = 0; i < m_num_vcs; i++) {
  74         m_out_vc_state.push_back(new OutVcState_d(i, m_net_ptr));
  75     }
  76 }
  77
  78 NetworkInterface_d::~NetworkInterface_d()
  79 {
  80     deletePointers(m_out_vc_state);
  81     deletePointers(m_ni_buffers);
  82     delete creditQueue;
  83     delete outSrcQueue;
  84 }
  85
  86 void
  87 NetworkInterface_d::addInPort(NetworkLink_d *in_link,
  88                               CreditLink_d *credit_link)
  89 {
  90     inNetLink = in_link;
  91     in_link->setLinkConsumer(this);
  92     m_ni_credit_link = credit_link;
  93     credit_link->setSourceQueue(creditQueue);
  94 }
  95
  96 void
  97 NetworkInterface_d::addOutPort(NetworkLink_d *out_link,
  98                                CreditLink_d *credit_link)
  99 {
 100     m_credit_link = credit_link;
 101     credit_link->setLinkConsumer(this);
 102
 103     outNetLink = out_link;
 104     outSrcQueue = new flitBuffer_d();
 105     out_link->setSourceQueue(outSrcQueue);
 106 }
 107
 108 void
 109 NetworkInterface_d::addNode(map<int, MessageBuffer *>& in,
 110                             map<int, MessageBuffer *>& out)
 111 {
 112     inNode_ptr = in;
 113     outNode_ptr = out;
 114
 115     for (auto& it : in) {
 116         // the protocol injects messages into the NI
 117         it.second->setConsumer(this);
 118         it.second->setReceiver(this);
 119     }
 120
 121     for (auto& it : out) {
 122         it.second->setSender(this);
 123     }
 124 }
 125
 126 bool
 127 NetworkInterface_d::flitisizeMessage(MsgPtr msg_ptr, int vnet)
 128 {
 129     NetworkMessage *net_msg_ptr = safe_cast<NetworkMessage *>(msg_ptr.get());
 130     NetDest net_msg_dest = net_msg_ptr->getInternalDestination();
 131
 132     // gets all the destinations associated with this message.
 133     vector<NodeID> dest_nodes = net_msg_dest.getAllDest();
 134
 135     // Number of flits is dependent on the link bandwidth available.
 136     // This is expressed in terms of bytes/cycle or the flit size
 137     int num_flits = (int) ceil((double) m_net_ptr->MessageSizeType_to_int(
 138         net_msg_ptr->getMessageSize())/m_net_ptr->getNiFlitSize());
 139
 140     // loop to convert all multicast messages into unicast messages
 141     for (int ctr = 0; ctr < dest_nodes.size(); ctr++) {
 142
 143         // this will return a free output virtual channel
 144         int vc = calculateVC(vnet);
 145
 146         if (vc == -1) {
 147             return false ;
 148         }
 149         MsgPtr new_msg_ptr = msg_ptr->clone();
 150         NodeID destID = dest_nodes[ctr];
 151
 152         NetworkMessage *new_net_msg_ptr =
 153             safe_cast<NetworkMessage *>(new_msg_ptr.get());
 154         if (dest_nodes.size() > 1) {
 155             NetDest personal_dest;
 156             for (int m = 0; m < (int) MachineType_NUM; m++) {
 157                 if ((destID >= MachineType_base_number((MachineType) m)) &&
 158                     destID < MachineType_base_number((MachineType) (m+1))) {
 159                     // calculating the NetDest associated with this destID
 160                     personal_dest.clear();
 161                     personal_dest.add((MachineID) {(MachineType) m, (destID -
 162                         MachineType_base_number((MachineType) m))});
 163                     new_net_msg_ptr->getInternalDestination() = personal_dest;
 164                     break;
 165                 }
 166             }
 167             net_msg_dest.removeNetDest(personal_dest);
 168             // removing the destination from the original message to reflect
 169             // that a message with this particular destination has been
 170             // flitisized and an output vc is acquired
 171             net_msg_ptr->getInternalDestination().removeNetDest(personal_dest);
 172         }
 173
 174         for (int i = 0; i < num_flits; i++) {
 175             m_net_ptr->increment_injected_flits(vnet);
 176             flit_d *fl = new flit_d(i, vc, vnet, num_flits, new_msg_ptr,
 177                 curCycle());
 178
 179             fl->set_delay(curCycle() - ticksToCycles(msg_ptr->getTime()));
 180             m_ni_buffers[vc]->insert(fl);
 181         }
 182
 183         m_ni_enqueue_time[vc] = curCycle();
 184         m_out_vc_state[vc]->setState(ACTIVE_, curCycle());
 185     }
 186     return true ;
 187 }
 188
 189 // Looking for a free output vc
 190 int
 191 NetworkInterface_d::calculateVC(int vnet)
 192 {
 193         for (int i = 0; i < m_vc_per_vnet; i++) {
 194                 int delta = m_vc_allocator[vnet];
 195                 m_vc_allocator[vnet]++;
 196                 if(m_vc_allocator[vnet] == m_vc_per_vnet)
 197                         m_vc_allocator[vnet] = 0;
 198
 199                 if (m_out_vc_state[(vnet*m_vc_per_vnet) + delta]->isInState(
 200                     IDLE_, curCycle())) {
 201                         return ((vnet*m_vc_per_vnet) + delta);
 202                 }
 203         }
 204         return -1;
 205 }
 206
 207 /*
 208  * The NI wakeup checks whether there are any ready messages in the protocol
 209  * buffer. If yes, it picks that up, flitisizes it into a number of flits and
 210  * puts it into an output buffer and schedules the output link. On a wakeup
 211  * it also checks whether there are flits in the input link. If yes, it picks
 212  * them up and if the flit is a tail, the NI inserts the corresponding message
 213  * into the protocol buffer. It also checks for credits being sent by the
 214  * downstream router.
 215  */
 216
 217 void
 218 NetworkInterface_d::wakeup()
 219 {
 220     DPRINTF(RubyNetwork, "m_id: %d woke up at time: %lld", m_id, curCycle());
 221
 222     MsgPtr msg_ptr;
 223
 224     // Checking for messages coming from the protocol
 225     // can pick up a message/cycle for each virtual net
 226     for (auto it = inNode_ptr.begin(); it != inNode_ptr.end(); ++it) {
 227         int vnet = (*it).first;
 228         MessageBuffer *b = (*it).second;
 229
 230         while (b->isReady()) { // Is there a message waiting
 231             msg_ptr = b->peekMsgPtr();
 232             if (flitisizeMessage(msg_ptr, vnet)) {
 233                 b->dequeue();
 234             } else {
 235                 break;
 236             }
 237         }
 238     }
 239
 240     scheduleOutputLink();
 241     checkReschedule();
 242
 243     /*********** Picking messages destined for this NI **********/
 244
 245     if (inNetLink->isReady(curCycle())) {
 246         flit_d *t_flit = inNetLink->consumeLink();
 247         bool free_signal = false;
 248         if (t_flit->get_type() == TAIL_ || t_flit->get_type() == HEAD_TAIL_) {
 249             free_signal = true;
 250
 251             outNode_ptr[t_flit->get_vnet()]->enqueue(
 252                 t_flit->get_msg_ptr(), Cycles(1));
 253         }
 254         // Simply send a credit back since we are not buffering
 255         // this flit in the NI
 256         flit_d *credit_flit = new flit_d(t_flit->get_vc(), free_signal,
 257                                          curCycle());
 258         creditQueue->insert(credit_flit);
 259         m_ni_credit_link->
 260             scheduleEventAbsolute(clockEdge(Cycles(1)));
 261
 262         int vnet = t_flit->get_vnet();
 263         m_net_ptr->increment_received_flits(vnet);
 264         Cycles network_delay = curCycle() - t_flit->get_enqueue_time();
 265         Cycles queueing_delay = t_flit->get_delay();
 266
 267         m_net_ptr->increment_network_latency(network_delay, vnet);
 268         m_net_ptr->increment_queueing_latency(queueing_delay, vnet);
 269         delete t_flit;
 270     }
 271
 272     /****************** Checking for credit link *******/
 273
 274     if (m_credit_link->isReady(curCycle())) {
 275         flit_d *t_flit = m_credit_link->consumeLink();
 276         m_out_vc_state[t_flit->get_vc()]->increment_credit();
 277         if (t_flit->is_free_signal()) {
 278             m_out_vc_state[t_flit->get_vc()]->setState(IDLE_, curCycle());
 279         }
 280         delete t_flit;
 281     }
 282 }
 283
 284 /** This function looks at the NI buffers
 285  *  if some buffer has flits which are ready to traverse the link in the next
 286  *  cycle, and the downstream output vc associated with this flit has buffers
 287  *  left, the link is scheduled for the next cycle
 288  */
 289
 290 void
 291 NetworkInterface_d::scheduleOutputLink()
 292 {
 293     int vc = m_vc_round_robin;
 294     m_vc_round_robin++;
 295     if (m_vc_round_robin == m_num_vcs)
 296         m_vc_round_robin = 0;
 297
 298     for (int i = 0; i < m_num_vcs; i++) {
 299         vc++;
 300         if (vc == m_num_vcs)
 301             vc = 0;
 302
 303         // model buffer backpressure
 304         if (m_ni_buffers[vc]->isReady(curCycle()) &&
 305             m_out_vc_state[vc]->has_credits()) {
 306
 307             bool is_candidate_vc = true;
 308             int t_vnet = get_vnet(vc);
 309             int vc_base = t_vnet * m_vc_per_vnet;
 310
 311             if (m_net_ptr->isVNetOrdered(t_vnet)) {
 312                 for (int vc_offset = 0; vc_offset < m_vc_per_vnet;
 313                      vc_offset++) {
 314                     int t_vc = vc_base + vc_offset;
 315                     if (m_ni_buffers[t_vc]->isReady(curCycle())) {
 316                         if (m_ni_enqueue_time[t_vc] < m_ni_enqueue_time[vc]) {
 317                             is_candidate_vc = false;
 318                             break;
 319                         }
 320                     }
 321                 }
 322             }
 323             if (!is_candidate_vc)
 324                 continue;
 325
 326             m_out_vc_state[vc]->decrement_credit();
 327             // Just removing the flit
 328             flit_d *t_flit = m_ni_buffers[vc]->getTopFlit();
 329             t_flit->set_time(curCycle() + Cycles(1));
 330             outSrcQueue->insert(t_flit);
 331             // schedule the out link
 332             outNetLink->scheduleEventAbsolute(clockEdge(Cycles(1)));
 333
 334             if (t_flit->get_type() == TAIL_ ||
 335                t_flit->get_type() == HEAD_TAIL_) {
 336                 m_ni_enqueue_time[vc] = Cycles(INFINITE_);
 337             }
 338             return;
 339         }
 340     }
 341 }
 342
 343 int
 344 NetworkInterface_d::get_vnet(int vc)
 345 {
 346     for (int i = 0; i < m_virtual_networks; i++) {
 347         if (vc >= (i*m_vc_per_vnet) && vc < ((i+1)*m_vc_per_vnet)) {
 348             return i;
 349         }
 350     }
 351     fatal("Could not determine vc");
 352 }
 353
 354 void
 355 NetworkInterface_d::checkReschedule()
 356 {
 357     for (const auto& it : inNode_ptr) {
 358         MessageBuffer *b = it.second;
 359
 360         while (b->isReady()) { // Is there a message waiting
 361             scheduleEvent(Cycles(1));
 362             return;
 363         }
 364     }
 365
 366     for (int vc = 0; vc < m_num_vcs; vc++) {
 367         if (m_ni_buffers[vc]->isReady(curCycle() + Cycles(1))) {
 368             scheduleEvent(Cycles(1));
 369             return;
 370         }
 371     }
 372 }
 373
 374 void
 375 NetworkInterface_d::print(std::ostream& out) const
 376 {
 377     out << "[Network Interface]";
 378 }
 379
 380 uint32_t
 381 NetworkInterface_d::functionalWrite(Packet *pkt)
 382 {
 383     uint32_t num_functional_writes = 0;
 384     for (unsigned int i  = 0; i < m_num_vcs; ++i) {
 385         num_functional_writes += m_ni_buffers[i]->functionalWrite(pkt);
 386     }
 387
 388     num_functional_writes += outSrcQueue->functionalWrite(pkt);
 389     return num_functional_writes;
 390 }
 391
 392 NetworkInterface_d *
 393 GarnetNetworkInterface_dParams::create()
 394 {
 395     return new NetworkInterface_d(this);
 396 }
 397