src/mem/qos/mem_ctrl.hh

   1 /*
   2  * Copyright (c) 2018 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  *
  37  * Authors: Matteo Andreozzi
  38  */
  39
  40 #include "debug/QOS.hh"
  41 #include "mem/abstract_mem.hh"
  42 #include "mem/qos/q_policy.hh"
  43 #include "mem/qos/policy.hh"
  44 #include "params/QoSMemCtrl.hh"
  45 #include "sim/system.hh"
  46
  47 #include <unordered_map>
  48 #include <vector>
  49 #include <deque>
  50
  51 #ifndef __MEM_QOS_MEM_CTRL_HH__
  52 #define __MEM_QOS_MEM_CTRL_HH__
  53
  54 namespace QoS {
  55
  56 /**
  57  * The QoS::MemCtrl is a base class for Memory objects
  58  * which support QoS - it provides access to a set of QoS
  59  * scheduling policies
  60  */
  61 class MemCtrl: public AbstractMemory
  62 {
  63   public:
  64     /** Bus Direction */
  65     enum BusState { READ, WRITE };
  66
  67   protected:
  68     /** QoS Policy, assigns QoS priority to the incoming packets */
  69     const std::unique_ptr<Policy> policy;
  70
  71     /** QoS Bus Turnaround Policy: selects the bus direction (READ/WRITE) */
  72     const std::unique_ptr<TurnaroundPolicy> turnPolicy;
  73
  74     /** QoS Queue Policy: selects packet among same-priority queue */
  75     const std::unique_ptr<QueuePolicy> queuePolicy;
  76
  77     /** Number of configured QoS priorities */
  78     const uint8_t _numPriorities;
  79
  80     /** Enables QoS priority escalation */
  81     const bool qosPriorityEscalation;
  82
  83     /**
  84      * Enables QoS synchronized scheduling invokes the QoS scheduler
  85      * on all masters, at every packet arrival.
  86      */
  87     const bool qosSyncroScheduler;
  88
  89     /** Hash of master ID - master name */
  90     std::unordered_map<MasterID, const std::string> masters;
  91
  92     /** Hash of masters - number of packets queued per priority */
  93     std::unordered_map<MasterID, std::vector<uint64_t> > packetPriorities;
  94
  95     /** Hash of masters - address of request - queue of times of request */
  96     std::unordered_map<MasterID,
  97             std::unordered_map<uint64_t, std::deque<uint64_t>> > requestTimes;
  98
  99     /**
 100      * Vector of QoS priorities/last service time. Refreshed at every
 101      * qosSchedule call.
 102      */
 103     std::vector<Tick> serviceTick;
 104
 105     /** Read request packets queue length in #packets, per QoS priority */
 106     std::vector<uint64_t> readQueueSizes;
 107
 108     /** Write request packets queue length in #packets, per QoS priority */
 109     std::vector<uint64_t> writeQueueSizes;
 110
 111     /** Total read request packets queue length in #packets */
 112     uint64_t totalReadQueueSize;
 113
 114     /** Total write request packets queue length in #packets */
 115     uint64_t totalWriteQueueSize;
 116
 117     /**
 118      * Bus state used to control the read/write switching and drive
 119      * the scheduling of the next request.
 120      */
 121     BusState busState;
 122
 123     /** bus state for next request event triggered */
 124     BusState busStateNext;
 125
 126     /** per-master average QoS priority */
 127     Stats::VectorStandardDeviation avgPriority;
 128     /** per-master average QoS distance between assigned and queued values */
 129     Stats::VectorStandardDeviation avgPriorityDistance;
 130
 131     /** per-priority minimum latency */
 132     Stats::Vector priorityMinLatency;
 133     /** per-priority maximum latency */
 134     Stats::Vector priorityMaxLatency;
 135     /** Count the number of turnarounds READ to WRITE */
 136     Stats::Scalar numReadWriteTurnArounds;
 137     /** Count the number of turnarounds WRITE to READ */
 138     Stats::Scalar numWriteReadTurnArounds;
 139     /** Count the number of times bus staying in READ state */
 140     Stats::Scalar numStayReadState;
 141     /** Count the number of times bus staying in WRITE state */
 142     Stats::Scalar numStayWriteState;
 143
 144     /** registers statistics */
 145     void regStats() override;
 146
 147     /**
 148      * Initializes dynamically counters and
 149      * statistics for a given Master
 150      *
 151      * @param m_id the master ID
 152      */
 153     void addMaster(const MasterID m_id);
 154
 155     /**
 156      * Called upon receiving a request or
 157      * updates statistics and updates queues status
 158      *
 159      * @param dir request direction
 160      * @param m_id master id
 161      * @param qos packet qos value
 162      * @param addr packet address
 163      * @param entries number of entries to record
 164      */
 165     void logRequest(BusState dir, MasterID m_id, uint8_t qos,
 166                     Addr addr, uint64_t entries);
 167
 168     /**
 169      * Called upon receiving a response,
 170      * updates statistics and updates queues status
 171      *
 172      * @param dir response direction
 173      * @param m_id master id
 174      * @param qos packet qos value
 175      * @param addr packet address
 176      * @param entries number of entries to record
 177      * @param delay response delay
 178      */
 179     void logResponse(BusState dir, MasterID m_id, uint8_t qos,
 180                      Addr addr, uint64_t entries, double delay);
 181
 182     /**
 183      * Assign priority to a packet by executing
 184      * the configured QoS policy.
 185      *
 186      * @param queues_ptr list of pointers to packet queues
 187      * @param queue_entry_size size in bytes per each packet in the queue
 188      * @param pkt pointer to the Packet
 189      * @return a QoS priority value
 190      */
 191     template<typename Queues>
 192     uint8_t qosSchedule(std::initializer_list<Queues*> queues_ptr,
 193                         uint64_t queue_entry_size, const PacketPtr pkt);
 194
 195     using SimObject::schedule;
 196     uint8_t schedule(MasterID m_id, uint64_t data);
 197     uint8_t schedule(const PacketPtr pkt);
 198
 199     /**
 200      * Returns next bus direction (READ or WRITE)
 201      * based on configured policy.
 202      */
 203     BusState selectNextBusState();
 204
 205     /**
 206      * Set current bus direction (READ or WRITE)
 207      * from next selected one
 208      */
 209     void setCurrentBusState() { busState = busStateNext; }
 210
 211     /**
 212      * Record statistics on turnarounds based on
 213      * busStateNext and busState values
 214      */
 215     void recordTurnaroundStats();
 216
 217     /**
 218      * Escalates/demotes priority of all packets
 219      * belonging to the passed master to given
 220      * priority value
 221      *
 222      * @param queues list of pointers to packet queues
 223      * @param queue_entry_size size of an entry in the queue
 224      * @param m_id master whose packets priority will change
 225      * @param tgt_prio target priority value
 226      */
 227     template<typename Queues>
 228     void escalate(std::initializer_list<Queues*> queues,
 229                   uint64_t queue_entry_size,
 230                   MasterID m_id, uint8_t tgt_prio);
 231
 232     /**
 233      * Escalates/demotes priority of all packets
 234      * belonging to the passed master to given
 235      * priority value in a specified cluster of queues
 236      * (e.g. read queues or write queues) which is passed
 237      * as an argument to the function.
 238      * The curr_prio/tgt_prio parameters are queue selectors in the
 239      * queue cluster.
 240      *
 241      * @param queues reference to packet queues
 242      * @param queue_entry_size size of an entry in the queue
 243      * @param m_id master whose packets priority will change
 244      * @param curr_prio source queue priority value
 245      * @param tgt_prio target queue priority value
 246      */
 247     template<typename Queues>
 248     void escalateQueues(Queues& queues, uint64_t queue_entry_size,
 249                         MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio);
 250
 251   public:
 252     /**
 253      * QoS Memory base class
 254      *
 255      * @param p pointer to QoSMemCtrl parameters
 256      */
 257     MemCtrl(const QoSMemCtrlParams*);
 258
 259     virtual ~MemCtrl();
 260
 261     /**
 262      * Initializes this object
 263      */
 264     void init() override;
 265
 266     /**
 267      * Gets the current bus state
 268      *
 269      * @return current bus state
 270      */
 271     BusState getBusState() const { return busState; }
 272
 273     /**
 274      * Gets the next bus state
 275      *
 276      * @return next bus state
 277      */
 278     BusState getBusStateNext() const { return busStateNext; }
 279
 280     /**
 281      * hasMaster returns true if the selected master(ID) has
 282      * been registered in the memory controller, which happens if
 283      * the memory controller has received at least a packet from
 284      * that master.
 285      *
 286      * @param m_id master id to lookup
 287      * @return true if the memory controller has received a packet
 288      *         from the master, false otherwise.
 289      */
 290     bool hasMaster(MasterID m_id) const
 291     {
 292         return masters.find(m_id) != masters.end();
 293     }
 294
 295     /**
 296      * Gets a READ queue size
 297      *
 298      * @param prio QoS Priority of the queue
 299      * @return queue size in packets
 300      */
 301     uint64_t getReadQueueSize(const uint8_t prio) const
 302     { return readQueueSizes[prio]; }
 303
 304     /**
 305      * Gets a WRITE queue size
 306      *
 307      * @param prio QoS Priority of the queue
 308      * @return queue size in packets
 309      */
 310     uint64_t getWriteQueueSize(const uint8_t prio) const
 311     { return writeQueueSizes[prio]; }
 312
 313     /**
 314      * Gets the total combined READ queues size
 315      *
 316      * @return total queues size in packets
 317      */
 318     uint64_t getTotalReadQueueSize() const { return totalReadQueueSize; }
 319
 320     /**
 321      * Gets the total combined WRITE queues size
 322      *
 323      * @return total queues size in packets
 324      */
 325     uint64_t getTotalWriteQueueSize() const { return totalWriteQueueSize; }
 326
 327     /**
 328      * Gets the last service tick related to a QoS Priority
 329      *
 330      * @param prio QoS Priority
 331      * @return tick
 332      */
 333     Tick getServiceTick(const uint8_t prio) const { return serviceTick[prio]; }
 334
 335     /**
 336      * Gets the total number of priority levels in the
 337      * QoS memory controller.
 338      *
 339      * @return total number of priority levels
 340      */
 341     uint8_t numPriorities() const { return _numPriorities; }
 342 };
 343
 344 template<typename Queues>
 345 void
 346 MemCtrl::escalateQueues(Queues& queues, uint64_t queue_entry_size,
 347                         MasterID m_id, uint8_t curr_prio, uint8_t tgt_prio)
 348 {
 349     auto it = queues[curr_prio].begin();
 350     while (it != queues[curr_prio].end()) {
 351         // No packets left to move
 352         if (packetPriorities[m_id][curr_prio] == 0)
 353             break;
 354
 355         auto pkt = *it;
 356
 357         DPRINTF(QOS,
 358                 "QoSMemCtrl::escalate checking priority %d packet "
 359                 "m_id %d address %d\n", curr_prio,
 360                 pkt->masterId(), pkt->getAddr());
 361
 362         // Found a packet to move
 363         if (pkt->masterId() == m_id) {
 364
 365             uint64_t moved_entries = divCeil(pkt->getSize(),
 366                                              queue_entry_size);
 367
 368             DPRINTF(QOS,
 369                     "QoSMemCtrl::escalate Master %s [id %d] moving "
 370                     "packet addr %d size %d (p size %d) from priority %d "
 371                     "to priority %d - "
 372                     "this master packets %d (entries to move %d)\n",
 373                     masters[m_id], m_id, pkt->getAddr(),
 374                     pkt->getSize(),
 375                     queue_entry_size, curr_prio, tgt_prio,
 376                     packetPriorities[m_id][curr_prio], moved_entries);
 377
 378
 379             if (pkt->isRead()) {
 380                 panic_if(readQueueSizes[curr_prio] < moved_entries,
 381                          "QoSMemCtrl::escalate master %s negative READ "
 382                          "packets for priority %d",
 383                         masters[m_id], tgt_prio);
 384                 readQueueSizes[curr_prio] -= moved_entries;
 385                 readQueueSizes[tgt_prio] += moved_entries;
 386             } else if (pkt->isWrite()) {
 387                 panic_if(writeQueueSizes[curr_prio] < moved_entries,
 388                          "QoSMemCtrl::escalate master %s negative WRITE "
 389                          "packets for priority %d",
 390                         masters[m_id], tgt_prio);
 391                 writeQueueSizes[curr_prio] -= moved_entries;
 392                 writeQueueSizes[tgt_prio] += moved_entries;
 393             }
 394
 395             // Change QoS priority and move packet
 396             pkt->qosValue(tgt_prio);
 397             queues[tgt_prio].push_back(pkt);
 398
 399             // Erase element from source packet queue, this will
 400             // increment the iterator
 401             it = queues[curr_prio].erase(it);
 402             panic_if(packetPriorities[m_id][curr_prio] < moved_entries,
 403                      "QoSMemCtrl::escalate master %s negative packets "
 404                      "for priority %d",
 405                      masters[m_id], tgt_prio);
 406
 407             packetPriorities[m_id][curr_prio] -= moved_entries;
 408             packetPriorities[m_id][tgt_prio] += moved_entries;
 409         } else {
 410             // Increment iterator to next location in the queue
 411             it++;
 412         }
 413     }
 414 }
 415
 416 template<typename Queues>
 417 void
 418 MemCtrl::escalate(std::initializer_list<Queues*> queues,
 419                   uint64_t queue_entry_size,
 420                   MasterID m_id, uint8_t tgt_prio)
 421 {
 422     // If needed, initialize all counters and statistics
 423     // for this master
 424     addMaster(m_id);
 425
 426     DPRINTF(QOS,
 427             "QoSMemCtrl::escalate Master %s [id %d] to priority "
 428             "%d (currently %d packets)\n",masters[m_id], m_id, tgt_prio,
 429             packetPriorities[m_id][tgt_prio]);
 430
 431     for (uint8_t curr_prio = 0; curr_prio < numPriorities(); ++curr_prio) {
 432         // Skip target priority
 433         if (curr_prio == tgt_prio)
 434             continue;
 435
 436         // Process other priority packet
 437         while (packetPriorities[m_id][curr_prio] > 0) {
 438             DPRINTF(QOS,
 439                     "QoSMemCtrl::escalate MID %d checking priority %d "
 440                     "(packets %d)- current packets in prio %d:  %d\n"
 441                     "\t(source read %d source write %d target read %d, "
 442                     "target write %d)\n",
 443                     m_id, curr_prio, packetPriorities[m_id][curr_prio],
 444                     tgt_prio, packetPriorities[m_id][tgt_prio],
 445                     readQueueSizes[curr_prio],
 446                     writeQueueSizes[curr_prio], readQueueSizes[tgt_prio],
 447                     writeQueueSizes[tgt_prio]);
 448
 449             // Check both read and write queue
 450             for (auto q : queues) {
 451                 escalateQueues(*q, queue_entry_size, m_id,
 452                                curr_prio, tgt_prio);
 453             }
 454         }
 455     }
 456
 457     DPRINTF(QOS,
 458             "QoSMemCtrl::escalate Completed master %s [id %d] to priority %d "
 459             "(now %d packets)\n\t(total read %d, total write %d)\n",
 460             masters[m_id], m_id, tgt_prio, packetPriorities[m_id][tgt_prio],
 461             readQueueSizes[tgt_prio], writeQueueSizes[tgt_prio]);
 462 }
 463
 464 template<typename Queues>
 465 uint8_t
 466 MemCtrl::qosSchedule(std::initializer_list<Queues*> queues,
 467                      const uint64_t queue_entry_size,
 468                      const PacketPtr pkt)
 469 {
 470     // Schedule packet.
 471     uint8_t pkt_priority = schedule(pkt);
 472
 473     assert(pkt_priority < numPriorities());
 474
 475     pkt->qosValue(pkt_priority);
 476
 477     if (qosSyncroScheduler) {
 478         // Call the scheduling function on all other masters.
 479         for (const auto& m : masters) {
 480
 481             if (m.first == pkt->masterId())
 482                 continue;
 483
 484             uint8_t prio = schedule(m.first, 0);
 485
 486             if (qosPriorityEscalation) {
 487                 DPRINTF(QOS,
 488                         "QoSMemCtrl::qosSchedule: (syncro) escalating "
 489                         "MASTER %s to assigned priority %d\n",
 490                         _system->getMasterName(m.first),
 491                         prio);
 492                 escalate(queues, queue_entry_size, m.first, prio);
 493             }
 494         }
 495     }
 496
 497     if (qosPriorityEscalation) {
 498         DPRINTF(QOS,
 499                 "QoSMemCtrl::qosSchedule: escalating "
 500                 "MASTER %s to assigned priority %d\n",
 501                 _system->getMasterName(pkt->masterId()),
 502                 pkt_priority);
 503         escalate(queues, queue_entry_size, pkt->masterId(), pkt_priority);
 504     }
 505
 506     // Update last service tick for selected priority
 507     serviceTick[pkt_priority] = curTick();
 508
 509     return pkt_priority;
 510 }
 511
 512 } // namespace QoS
 513
 514 #endif /* __MEM_QOS_MEM_CTRL_HH__ */