3 * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met: redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer;
10 * redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution;
13 * neither the name of the copyright holders nor the names of its
14 * contributors may be used to endorse or promote products derived from
15 * this software without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 * Description: see Throttle.hh
37 #include "mem/ruby/network/simple/Throttle.hh"
38 #include "mem/ruby/buffers/MessageBuffer.hh"
39 #include "mem/ruby/network/Network.hh"
40 #include "mem/ruby/system/System.hh"
41 #include "mem/ruby/slicc_interface/NetworkMessage.hh"
42 #include "mem/protocol/Protocol.hh"
44 const int HIGH_RANGE
= 256;
45 const int ADJUST_INTERVAL
= 50000;
46 const int MESSAGE_SIZE_MULTIPLIER
= 1000;
47 //const int BROADCAST_SCALING = 4; // Have a 16p system act like a 64p systems
48 const int BROADCAST_SCALING
= 1;
49 const int PRIORITY_SWITCH_LIMIT
= 128;
51 static int network_message_to_size(NetworkMessage
* net_msg_ptr
);
53 extern std::ostream
* debug_cout_ptr
;
55 Throttle::Throttle(int sID
, NodeID node
, int link_latency
, int link_bandwidth_multiplier
)
57 init(node
, link_latency
, link_bandwidth_multiplier
);
61 Throttle::Throttle(NodeID node
, int link_latency
, int link_bandwidth_multiplier
)
63 init(node
, link_latency
, link_bandwidth_multiplier
);
67 void Throttle::init(NodeID node
, int link_latency
, int link_bandwidth_multiplier
)
72 ASSERT(link_bandwidth_multiplier
> 0);
73 m_link_bandwidth_multiplier
= link_bandwidth_multiplier
;
74 m_link_latency
= link_latency
;
76 m_wakeups_wo_switch
= 0;
80 void Throttle::clear()
82 for (int counter
= 0; counter
< m_vnets
; counter
++) {
83 m_in
[counter
]->clear();
84 m_out
[counter
]->clear();
88 void Throttle::addLinks(const Vector
<MessageBuffer
*>& in_vec
, const Vector
<MessageBuffer
*>& out_vec
)
90 assert(in_vec
.size() == out_vec
.size());
91 for (int i
=0; i
<in_vec
.size(); i
++) {
92 addVirtualNetwork(in_vec
[i
], out_vec
[i
]);
95 m_message_counters
.setSize(MessageSizeType_NUM
);
96 for (int i
=0; i
<MessageSizeType_NUM
; i
++) {
97 m_message_counters
[i
].setSize(in_vec
.size());
98 for (int j
=0; j
<m_message_counters
[i
].size(); j
++) {
99 m_message_counters
[i
][j
] = 0;
104 void Throttle::addVirtualNetwork(MessageBuffer
* in_ptr
, MessageBuffer
* out_ptr
)
106 m_units_remaining
.insertAtBottom(0);
107 m_in
.insertAtBottom(in_ptr
);
108 m_out
.insertAtBottom(out_ptr
);
110 // Set consumer and description
111 m_in
[m_vnets
]->setConsumer(this);
112 string desc
= "[Queue to Throttle " + NodeIDToString(m_sID
) + " " + NodeIDToString(m_node
) + "]";
113 m_in
[m_vnets
]->setDescription(desc
);
117 void Throttle::wakeup()
119 // Limits the number of message sent to a limited number of bytes/cycle.
120 assert(getLinkBandwidth() > 0);
121 int bw_remaining
= getLinkBandwidth();
123 // Give the highest numbered link priority most of the time
124 m_wakeups_wo_switch
++;
125 int highest_prio_vnet
= m_vnets
-1;
126 int lowest_prio_vnet
= 0;
128 bool schedule_wakeup
= false;
130 // invert priorities to avoid starvation seen in the component network
131 if (m_wakeups_wo_switch
> PRIORITY_SWITCH_LIMIT
) {
132 m_wakeups_wo_switch
= 0;
133 highest_prio_vnet
= 0;
134 lowest_prio_vnet
= m_vnets
-1;
138 for (int vnet
= highest_prio_vnet
; (vnet
*counter
) >= (counter
*lowest_prio_vnet
); vnet
-= counter
) {
140 assert(m_out
[vnet
] != NULL
);
141 assert(m_in
[vnet
] != NULL
);
142 assert(m_units_remaining
[vnet
] >= 0);
144 while ((bw_remaining
> 0) && ((m_in
[vnet
]->isReady()) || (m_units_remaining
[vnet
] > 0)) && m_out
[vnet
]->areNSlotsAvailable(1)) {
146 // See if we are done transferring the previous message on this virtual network
147 if (m_units_remaining
[vnet
] == 0 && m_in
[vnet
]->isReady()) {
149 // Find the size of the message we are moving
150 MsgPtr msg_ptr
= m_in
[vnet
]->peekMsgPtr();
151 NetworkMessage
* net_msg_ptr
= dynamic_cast<NetworkMessage
*>(msg_ptr
.ref());
152 m_units_remaining
[vnet
] += network_message_to_size(net_msg_ptr
);
154 DEBUG_NEWLINE(NETWORK_COMP
,HighPrio
);
155 DEBUG_MSG(NETWORK_COMP
,HighPrio
,"throttle: " + int_to_string(m_node
)
156 + " my bw " + int_to_string(getLinkBandwidth())
157 + " bw spent enqueueing net msg " + int_to_string(m_units_remaining
[vnet
])
158 + " time: " + int_to_string(g_eventQueue_ptr
->getTime()) + ".");
161 m_out
[vnet
]->enqueue(m_in
[vnet
]->peekMsgPtr(), m_link_latency
);
165 m_message_counters
[net_msg_ptr
->getMessageSize()][vnet
]++;
167 DEBUG_MSG(NETWORK_COMP
,LowPrio
,*m_out
[vnet
]);
168 DEBUG_NEWLINE(NETWORK_COMP
,HighPrio
);
171 // Calculate the amount of bandwidth we spent on this message
172 int diff
= m_units_remaining
[vnet
] - bw_remaining
;
173 m_units_remaining
[vnet
] = max(0, diff
);
174 bw_remaining
= max(0, -diff
);
177 if ((bw_remaining
> 0) && ((m_in
[vnet
]->isReady()) || (m_units_remaining
[vnet
] > 0)) && !m_out
[vnet
]->areNSlotsAvailable(1)) {
178 DEBUG_MSG(NETWORK_COMP
,LowPrio
,vnet
);
179 schedule_wakeup
= true; // schedule me to wakeup again because I'm waiting for my output queue to become available
183 // We should only wake up when we use the bandwidth
184 // assert(bw_remaining != getLinkBandwidth()); // This is only mostly true
186 // Record that we used some or all of the link bandwidth this cycle
187 double ratio
= 1.0-(double(bw_remaining
)/double(getLinkBandwidth()));
188 // If ratio = 0, we used no bandwidth, if ratio = 1, we used all
191 if ((bw_remaining
> 0) && !schedule_wakeup
) {
192 // We have extra bandwidth and our output buffer was available, so we must not have anything else to do until another message arrives.
193 DEBUG_MSG(NETWORK_COMP
,LowPrio
,*this);
194 DEBUG_MSG(NETWORK_COMP
,LowPrio
,"not scheduled again");
196 DEBUG_MSG(NETWORK_COMP
,LowPrio
,*this);
197 DEBUG_MSG(NETWORK_COMP
,LowPrio
,"scheduled again");
198 // We are out of bandwidth for this cycle, so wakeup next cycle and continue
199 g_eventQueue_ptr
->scheduleEvent(this, 1);
203 void Throttle::printStats(ostream
& out
) const
205 out
<< "utilized_percent: " << getUtilization() << endl
;
208 void Throttle::clearStats()
210 m_ruby_start
= g_eventQueue_ptr
->getTime();
211 m_links_utilized
= 0.0;
213 for (int i
=0; i
<m_message_counters
.size(); i
++) {
214 for (int j
=0; j
<m_message_counters
[i
].size(); j
++) {
215 m_message_counters
[i
][j
] = 0;
220 void Throttle::printConfig(ostream
& out
) const
225 double Throttle::getUtilization() const
227 return (100.0 * double(m_links_utilized
)) / (double(g_eventQueue_ptr
->getTime()-m_ruby_start
));
230 void Throttle::print(ostream
& out
) const
232 out
<< "[Throttle: " << m_sID
<< " " << m_node
<< " bw: " << getLinkBandwidth() << "]";
238 int network_message_to_size(NetworkMessage
* net_msg_ptr
)
240 assert(net_msg_ptr
!= NULL
);
242 // Artificially increase the size of broadcast messages
243 if (BROADCAST_SCALING
> 1) {
244 if (net_msg_ptr
->getDestination().isBroadcast()) {
245 return (RubySystem::getNetwork()->MessageSizeType_to_int(net_msg_ptr
->getMessageSize()) * MESSAGE_SIZE_MULTIPLIER
* BROADCAST_SCALING
);
248 return (RubySystem::getNetwork()->MessageSizeType_to_int(net_msg_ptr
->getMessageSize()) * MESSAGE_SIZE_MULTIPLIER
);