2 * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "base/cast.hh"
32 #include "base/cprintf.hh"
33 #include "debug/RubyNetwork.hh"
34 #include "mem/ruby/buffers/MessageBuffer.hh"
35 #include "mem/ruby/network/simple/Throttle.hh"
36 #include "mem/ruby/network/Network.hh"
37 #include "mem/ruby/slicc_interface/NetworkMessage.hh"
38 #include "mem/ruby/system/System.hh"
42 const int HIGH_RANGE
= 256;
43 const int ADJUST_INTERVAL
= 50000;
44 const int MESSAGE_SIZE_MULTIPLIER
= 1000;
45 //const int BROADCAST_SCALING = 4; // Have a 16p system act like a 64p systems
46 const int BROADCAST_SCALING
= 1;
47 const int PRIORITY_SWITCH_LIMIT
= 128;
49 static int network_message_to_size(NetworkMessage
* net_msg_ptr
);
51 Throttle::Throttle(int sID
, NodeID node
, int link_latency
,
52 int link_bandwidth_multiplier
, int endpoint_bandwidth
)
54 init(node
, link_latency
, link_bandwidth_multiplier
, endpoint_bandwidth
);
58 Throttle::Throttle(NodeID node
, int link_latency
,
59 int link_bandwidth_multiplier
, int endpoint_bandwidth
)
61 init(node
, link_latency
, link_bandwidth_multiplier
, endpoint_bandwidth
);
66 Throttle::init(NodeID node
, int link_latency
, int link_bandwidth_multiplier
,
67 int endpoint_bandwidth
)
72 assert(link_bandwidth_multiplier
> 0);
73 m_link_bandwidth_multiplier
= link_bandwidth_multiplier
;
74 m_link_latency
= link_latency
;
75 m_endpoint_bandwidth
= endpoint_bandwidth
;
77 m_wakeups_wo_switch
= 0;
84 for (int counter
= 0; counter
< m_vnets
; counter
++) {
85 m_in
[counter
]->clear();
86 m_out
[counter
]->clear();
91 Throttle::addLinks(const std::vector
<MessageBuffer
*>& in_vec
,
92 const std::vector
<MessageBuffer
*>& out_vec
)
94 assert(in_vec
.size() == out_vec
.size());
95 for (int i
=0; i
<in_vec
.size(); i
++) {
96 addVirtualNetwork(in_vec
[i
], out_vec
[i
]);
99 m_message_counters
.resize(MessageSizeType_NUM
);
100 for (int i
= 0; i
< MessageSizeType_NUM
; i
++) {
101 m_message_counters
[i
].resize(in_vec
.size());
102 for (int j
= 0; j
<m_message_counters
[i
].size(); j
++) {
103 m_message_counters
[i
][j
] = 0;
109 Throttle::addVirtualNetwork(MessageBuffer
* in_ptr
, MessageBuffer
* out_ptr
)
111 m_units_remaining
.push_back(0);
112 m_in
.push_back(in_ptr
);
113 m_out
.push_back(out_ptr
);
115 // Set consumer and description
116 m_in
[m_vnets
]->setConsumer(this);
117 string desc
= "[Queue to Throttle " + to_string(m_sID
) + " " +
118 to_string(m_node
) + "]";
119 m_in
[m_vnets
]->setDescription(desc
);
126 // Limits the number of message sent to a limited number of bytes/cycle.
127 assert(getLinkBandwidth() > 0);
128 int bw_remaining
= getLinkBandwidth();
130 // Give the highest numbered link priority most of the time
131 m_wakeups_wo_switch
++;
132 int highest_prio_vnet
= m_vnets
-1;
133 int lowest_prio_vnet
= 0;
135 bool schedule_wakeup
= false;
137 // invert priorities to avoid starvation seen in the component network
138 if (m_wakeups_wo_switch
> PRIORITY_SWITCH_LIMIT
) {
139 m_wakeups_wo_switch
= 0;
140 highest_prio_vnet
= 0;
141 lowest_prio_vnet
= m_vnets
-1;
145 for (int vnet
= highest_prio_vnet
;
146 (vnet
* counter
) >= (counter
* lowest_prio_vnet
);
149 assert(m_out
[vnet
] != NULL
);
150 assert(m_in
[vnet
] != NULL
);
151 assert(m_units_remaining
[vnet
] >= 0);
153 while (bw_remaining
> 0 &&
154 (m_in
[vnet
]->isReady() || m_units_remaining
[vnet
] > 0) &&
155 m_out
[vnet
]->areNSlotsAvailable(1)) {
157 // See if we are done transferring the previous message on
158 // this virtual network
159 if (m_units_remaining
[vnet
] == 0 && m_in
[vnet
]->isReady()) {
160 // Find the size of the message we are moving
161 MsgPtr msg_ptr
= m_in
[vnet
]->peekMsgPtr();
162 NetworkMessage
* net_msg_ptr
=
163 safe_cast
<NetworkMessage
*>(msg_ptr
.get());
164 m_units_remaining
[vnet
] +=
165 network_message_to_size(net_msg_ptr
);
167 DPRINTF(RubyNetwork
, "throttle: %d my bw %d bw spent "
168 "enqueueing net msg %d time: %lld.\n",
169 m_node
, getLinkBandwidth(), m_units_remaining
[vnet
],
170 g_eventQueue_ptr
->getTime());
173 m_out
[vnet
]->enqueue(m_in
[vnet
]->peekMsgPtr(), m_link_latency
);
177 m_message_counters
[net_msg_ptr
->getMessageSize()][vnet
]++;
179 DPRINTF(RubyNetwork
, "%s\n", *m_out
[vnet
]);
182 // Calculate the amount of bandwidth we spent on this message
183 int diff
= m_units_remaining
[vnet
] - bw_remaining
;
184 m_units_remaining
[vnet
] = max(0, diff
);
185 bw_remaining
= max(0, -diff
);
188 if (bw_remaining
> 0 &&
189 (m_in
[vnet
]->isReady() || m_units_remaining
[vnet
] > 0) &&
190 !m_out
[vnet
]->areNSlotsAvailable(1)) {
191 DPRINTF(RubyNetwork
, "vnet: %d", vnet
);
192 // schedule me to wakeup again because I'm waiting for my
193 // output queue to become available
194 schedule_wakeup
= true;
198 // We should only wake up when we use the bandwidth
199 // This is only mostly true
200 // assert(bw_remaining != getLinkBandwidth());
202 // Record that we used some or all of the link bandwidth this cycle
203 double ratio
= 1.0 - (double(bw_remaining
) / double(getLinkBandwidth()));
205 // If ratio = 0, we used no bandwidth, if ratio = 1, we used all
208 if (bw_remaining
> 0 && !schedule_wakeup
) {
209 // We have extra bandwidth and our output buffer was
210 // available, so we must not have anything else to do until
211 // another message arrives.
212 DPRINTF(RubyNetwork
, "%s not scheduled again\n", *this);
214 DPRINTF(RubyNetwork
, "%s scheduled again\n", *this);
216 // We are out of bandwidth for this cycle, so wakeup next
217 // cycle and continue
218 g_eventQueue_ptr
->scheduleEvent(this, 1);
223 Throttle::printStats(ostream
& out
) const
225 out
<< "utilized_percent: " << getUtilization() << endl
;
229 Throttle::clearStats()
231 m_ruby_start
= g_eventQueue_ptr
->getTime();
232 m_links_utilized
= 0.0;
234 for (int i
= 0; i
< m_message_counters
.size(); i
++) {
235 for (int j
= 0; j
< m_message_counters
[i
].size(); j
++) {
236 m_message_counters
[i
][j
] = 0;
242 Throttle::getUtilization() const
244 return 100.0 * double(m_links_utilized
) /
245 double(g_eventQueue_ptr
->getTime()-m_ruby_start
);
249 Throttle::print(ostream
& out
) const
251 ccprintf(out
, "[%i bw: %i]", m_node
, getLinkBandwidth());
255 network_message_to_size(NetworkMessage
* net_msg_ptr
)
257 assert(net_msg_ptr
!= NULL
);
259 int size
= RubySystem::getNetwork()->
260 MessageSizeType_to_int(net_msg_ptr
->getMessageSize());
261 size
*= MESSAGE_SIZE_MULTIPLIER
;
263 // Artificially increase the size of broadcast messages
264 if (BROADCAST_SCALING
> 1 && net_msg_ptr
->getDestination().isBroadcast())
265 size
*= BROADCAST_SCALING
;