2 * Copyright (c) 2008 Princeton University
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 * Authors: Niket Agarwal
33 #include "base/stl_helpers.hh"
34 #include "mem/ruby/network/garnet/fixed-pipeline/NetworkInterface_d.hh"
35 #include "mem/ruby/buffers/MessageBuffer.hh"
36 #include "mem/ruby/network/garnet/fixed-pipeline/flitBuffer_d.hh"
37 #include "mem/ruby/slicc_interface/NetworkMessage.hh"
40 using m5::stl_helpers::deletePointers
;
42 NetworkInterface_d::NetworkInterface_d(int id
, int virtual_networks
, GarnetNetwork_d
*network_ptr
)
45 m_net_ptr
= network_ptr
;
46 m_virtual_networks
= virtual_networks
;
47 m_vc_per_vnet
= m_net_ptr
->getVCsPerClass();
48 m_num_vcs
= m_vc_per_vnet
*m_virtual_networks
;
51 m_ni_buffers
.resize(m_num_vcs
);
52 m_ni_enqueue_time
.resize(m_num_vcs
);
53 inNode_ptr
.resize(m_virtual_networks
);
54 outNode_ptr
.resize(m_virtual_networks
);
55 creditQueue
= new flitBuffer_d();
57 for(int i
=0; i
< m_num_vcs
; i
++)
59 m_ni_buffers
[i
] = new flitBuffer_d(); // instantiating the NI flit buffers
60 m_ni_enqueue_time
[i
] = INFINITE_
;
62 m_vc_allocator
.resize(m_virtual_networks
); // 1 allocator per virtual net
63 for(int i
= 0; i
< m_virtual_networks
; i
++)
65 m_vc_allocator
[i
] = 0;
68 for(int i
= 0; i
< m_num_vcs
; i
++)
70 m_out_vc_state
.push_back(new OutVcState_d(i
, m_net_ptr
));
71 m_out_vc_state
[i
]->setState(IDLE_
, g_eventQueue_ptr
->getTime());
75 NetworkInterface_d::~NetworkInterface_d()
77 deletePointers(m_out_vc_state
);
78 deletePointers(m_ni_buffers
);
83 void NetworkInterface_d::addInPort(NetworkLink_d
*in_link
, CreditLink_d
*credit_link
)
86 in_link
->setLinkConsumer(this);
87 m_ni_credit_link
= credit_link
;
88 credit_link
->setSourceQueue(creditQueue
);
91 void NetworkInterface_d::addOutPort(NetworkLink_d
*out_link
, CreditLink_d
*credit_link
)
93 m_credit_link
= credit_link
;
94 credit_link
->setLinkConsumer(this);
96 outNetLink
= out_link
;
97 outSrcQueue
= new flitBuffer_d();
98 out_link
->setSourceQueue(outSrcQueue
);
101 void NetworkInterface_d::addNode(vector
<MessageBuffer
*>& in
, vector
<MessageBuffer
*>& out
)
103 ASSERT(in
.size() == m_virtual_networks
);
106 for (int j
= 0; j
< m_virtual_networks
; j
++)
108 inNode_ptr
[j
]->setConsumer(this); // So that protocol injects messages into the NI
112 bool NetworkInterface_d::flitisizeMessage(MsgPtr msg_ptr
, int vnet
)
114 NetworkMessage
*net_msg_ptr
=
115 safe_cast
<NetworkMessage
*>(msg_ptr
.get());
116 NetDest net_msg_dest
= net_msg_ptr
->getInternalDestination();
117 vector
<NodeID
> dest_nodes
= net_msg_dest
.getAllDest(); // gets all the destinations associated with this message.
119 int num_flits
= (int) ceil((double) m_net_ptr
->MessageSizeType_to_int(net_msg_ptr
->getMessageSize())/m_net_ptr
->getFlitSize() ); // Number of flits is dependent on the link bandwidth available. This is expressed in terms of bytes/cycle or the flit size
121 for(int ctr
= 0; ctr
< dest_nodes
.size(); ctr
++) // loop because we will be converting all multicast messages into unicast messages
123 int vc
= calculateVC(vnet
); // this will return a free output virtual channel
128 MsgPtr new_msg_ptr
= msg_ptr
->clone();
129 NodeID destID
= dest_nodes
[ctr
];
131 NetworkMessage
*new_net_msg_ptr
=
132 safe_cast
<NetworkMessage
*>(new_msg_ptr
.get());
133 if(dest_nodes
.size() > 1)
135 NetDest personal_dest
;
136 for(int m
= 0; m
< (int) MachineType_NUM
; m
++)
138 if((destID
>= MachineType_base_number((MachineType
) m
)) && destID
< MachineType_base_number((MachineType
) (m
+1)))
140 // calculating the NetDest associated with this destination ID
141 personal_dest
.clear();
142 personal_dest
.add((MachineID
) {(MachineType
) m
, (destID
- MachineType_base_number((MachineType
) m
))});
143 new_net_msg_ptr
->getInternalDestination() = personal_dest
;
147 net_msg_dest
.removeNetDest(personal_dest
);
148 net_msg_ptr
->getInternalDestination().removeNetDest(personal_dest
); // removing the destination from the original message to reflect that a message with this particular destination has been flitisized and an output vc is acquired
150 for(int i
= 0; i
< num_flits
; i
++)
152 m_net_ptr
->increment_injected_flits();
153 flit_d
*fl
= new flit_d(i
, vc
, vnet
, num_flits
, new_msg_ptr
);
154 fl
->set_delay(g_eventQueue_ptr
->getTime() - msg_ptr
->getTime());
155 m_ni_buffers
[vc
]->insert(fl
);
157 m_ni_enqueue_time
[vc
] = g_eventQueue_ptr
->getTime();
158 m_out_vc_state
[vc
]->setState(ACTIVE_
, g_eventQueue_ptr
->getTime());
163 // Looking for a free output vc
164 int NetworkInterface_d::calculateVC(int vnet
)
166 for(int i
= 0; i
< m_vc_per_vnet
; i
++)
168 int delta
= m_vc_allocator
[vnet
];
169 m_vc_allocator
[vnet
]++;
170 if(m_vc_allocator
[vnet
] == m_vc_per_vnet
)
171 m_vc_allocator
[vnet
] = 0;
173 if(m_out_vc_state
[(vnet
*m_vc_per_vnet
) + delta
]->isInState(IDLE_
, g_eventQueue_ptr
->getTime()))
175 return ((vnet
*m_vc_per_vnet
) + delta
);
182 * The NI wakeup checks whether there are any ready messages in the protocol buffer. If yes,
183 * it picks that up, flitisizes it into a number of flits and puts it into an output buffer
184 * and schedules the output link. On a wakeup it also checks whether there are flits in the
185 * input link. If yes, it picks them up and if the flit is a tail, the NI inserts the
186 * corresponding message into the protocol buffer. It also checks for credits being sent
187 * by the downstream router.
190 void NetworkInterface_d::wakeup()
192 DEBUG_EXPR(NETWORK_COMP
, MedPrio
, m_id
);
193 DEBUG_MSG(NETWORK_COMP
, MedPrio
, "NI WOKE UP");
194 DEBUG_EXPR(NETWORK_COMP
, MedPrio
, g_eventQueue_ptr
->getTime());
198 //Checking for messages coming from the protocol
199 for (int vnet
= 0; vnet
< m_virtual_networks
; vnet
++) // can pick up a message/cycle for each virtual net
201 while(inNode_ptr
[vnet
]->isReady()) // Is there a message waiting
203 msg_ptr
= inNode_ptr
[vnet
]->peekMsgPtr();
204 if(flitisizeMessage(msg_ptr
, vnet
))
206 inNode_ptr
[vnet
]->pop();
215 scheduleOutputLink();
218 /*********** Picking messages destined for this NI **********/
220 if(inNetLink
->isReady())
222 flit_d
*t_flit
= inNetLink
->consumeLink();
223 bool free_signal
= false;
224 if(t_flit
->get_type() == TAIL_
|| t_flit
->get_type() == HEAD_TAIL_
)
227 if(!m_net_ptr
->isNetworkTesting()) // When we are doing network only testing, the messages do not have to be buffered into the message buffers
229 outNode_ptr
[t_flit
->get_vnet()]->enqueue(t_flit
->get_msg_ptr(), 1); // enqueueing for protocol buffer. This is not required when doing network only testing
232 flit_d
*credit_flit
= new flit_d(t_flit
->get_vc(), free_signal
); // Simply send a credit back since we are not buddering this flit in the NI
233 creditQueue
->insert(credit_flit
);
234 g_eventQueue_ptr
->scheduleEvent(m_ni_credit_link
, 1);
236 m_net_ptr
->increment_recieved_flits();
237 int network_delay
= g_eventQueue_ptr
->getTime() - t_flit
->get_enqueue_time();
238 int queueing_delay
= t_flit
->get_delay();
239 m_net_ptr
->increment_network_latency(network_delay
);
240 m_net_ptr
->increment_queueing_latency(queueing_delay
);
244 /****************** Checking for credit link *******/
246 if(m_credit_link
->isReady())
248 flit_d
*t_flit
= m_credit_link
->consumeLink();
249 m_out_vc_state
[t_flit
->get_vc()]->increment_credit();
250 if(t_flit
->is_free_signal())
252 m_out_vc_state
[t_flit
->get_vc()]->setState(IDLE_
, g_eventQueue_ptr
->getTime());
258 // This function look at the NI buffers and if some buffer has flits which are ready to traverse the link in the next cycle and also the downstream output vc associated with this flit has buffers left, the link is scheduled for the next cycle
260 void NetworkInterface_d::scheduleOutputLink()
262 int vc
= m_vc_round_robin
;
264 if(m_vc_round_robin
== m_num_vcs
)
265 m_vc_round_robin
= 0;
267 for(int i
= 0; i
< m_num_vcs
; i
++)
272 if(m_ni_buffers
[vc
]->isReady() && m_out_vc_state
[vc
]->has_credits()) // models buffer backpressure
274 bool is_candidate_vc
= true;
275 int t_vnet
= get_vnet(vc
);
276 int vc_base
= t_vnet
* m_vc_per_vnet
;
278 if(m_net_ptr
->isVNetOrdered(t_vnet
))
280 for (int vc_offset
= 0; vc_offset
< m_vc_per_vnet
; vc_offset
++)
282 int t_vc
= vc_base
+ vc_offset
;
283 if(m_ni_buffers
[t_vc
]->isReady())
285 if(m_ni_enqueue_time
[t_vc
] < m_ni_enqueue_time
[vc
])
287 is_candidate_vc
= false;
296 m_out_vc_state
[vc
]->decrement_credit();
297 flit_d
*t_flit
= m_ni_buffers
[vc
]->getTopFlit(); // Just removing the flit
298 t_flit
->set_time(g_eventQueue_ptr
->getTime() + 1);
299 outSrcQueue
->insert(t_flit
);
300 g_eventQueue_ptr
->scheduleEvent(outNetLink
, 1); // schedule the out link
302 if(t_flit
->get_type() == TAIL_
|| t_flit
->get_type() == HEAD_TAIL_
)
304 m_ni_enqueue_time
[vc
] = INFINITE_
;
311 int NetworkInterface_d::get_vnet(int vc
)
313 for(int i
= 0; i
< m_net_ptr
->getNumberOfVirtualNetworks(); i
++)
315 if(vc
>= (i
*m_vc_per_vnet
) && vc
< ((i
+1)*m_vc_per_vnet
))
320 ERROR_MSG("Could not determine vc");
324 void NetworkInterface_d::checkReschedule()
326 for(int vnet
= 0; vnet
< m_virtual_networks
; vnet
++)
328 if(inNode_ptr
[vnet
]->isReady()) // Is there a message waiting
330 g_eventQueue_ptr
->scheduleEvent(this, 1);
334 for(int vc
= 0; vc
< m_num_vcs
; vc
++)
336 if(m_ni_buffers
[vc
]->isReadyForNext())
338 g_eventQueue_ptr
->scheduleEvent(this, 1);
344 void NetworkInterface_d::printConfig(std::ostream
& out
) const
346 out
<< "[Network Interface " << m_id
<< "] - ";
347 out
<< "[inLink " << inNetLink
->get_id() << "] - ";
348 out
<< "[outLink " << outNetLink
->get_id() << "]" << std::endl
;
351 void NetworkInterface_d::print(std::ostream
& out
) const
353 out
<< "[Network Interface]";