src/mem/qos/mem_ctrl.hh

   1 /*
   2  * Copyright (c) 2020 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include "debug/QOS.hh"
  39 #include "mem/qos/policy.hh"
  40 #include "mem/qos/q_policy.hh"
  41 #include "params/QoSMemCtrl.hh"
  42 #include "sim/clocked_object.hh"
  43 #include "sim/system.hh"
  44
  45 #include <unordered_map>
  46 #include <vector>
  47 #include <deque>
  48
  49 #ifndef __MEM_QOS_MEM_CTRL_HH__
  50 #define __MEM_QOS_MEM_CTRL_HH__
  51
  52 namespace QoS {
  53
  54 /**
  55  * The QoS::MemCtrl is a base class for Memory objects
  56  * which support QoS - it provides access to a set of QoS
  57  * scheduling policies
  58  */
  59 class MemCtrl : public ClockedObject
  60 {
  61   public:
  62     /** Bus Direction */
  63     enum BusState { READ, WRITE };
  64
  65   protected:
  66     /** QoS Policy, assigns QoS priority to the incoming packets */
  67     const std::unique_ptr<Policy> policy;
  68
  69     /** QoS Bus Turnaround Policy: selects the bus direction (READ/WRITE) */
  70     const std::unique_ptr<TurnaroundPolicy> turnPolicy;
  71
  72     /** QoS Queue Policy: selects packet among same-priority queue */
  73     const std::unique_ptr<QueuePolicy> queuePolicy;
  74
  75     /** Number of configured QoS priorities */
  76     const uint8_t _numPriorities;
  77
  78     /** Enables QoS priority escalation */
  79     const bool qosPriorityEscalation;
  80
  81     /**
  82      * Enables QoS synchronized scheduling invokes the QoS scheduler
  83      * on all requestors, at every packet arrival.
  84      */
  85     const bool qosSyncroScheduler;
  86
  87     /** Hash of requestor ID - requestor name */
  88     std::unordered_map<RequestorID, const std::string> requestors;
  89
  90     /** Hash of requestors - number of packets queued per priority */
  91     std::unordered_map<RequestorID, std::vector<uint64_t> > packetPriorities;
  92
  93     /** Hash of requestors - address of request - queue of times of request */
  94     std::unordered_map<RequestorID,
  95             std::unordered_map<uint64_t, std::deque<uint64_t>> > requestTimes;
  96
  97     /**
  98      * Vector of QoS priorities/last service time. Refreshed at every
  99      * qosSchedule call.
 100      */
 101     std::vector<Tick> serviceTick;
 102
 103     /** Read request packets queue length in #packets, per QoS priority */
 104     std::vector<uint64_t> readQueueSizes;
 105
 106     /** Write request packets queue length in #packets, per QoS priority */
 107     std::vector<uint64_t> writeQueueSizes;
 108
 109     /** Total read request packets queue length in #packets */
 110     uint64_t totalReadQueueSize;
 111
 112     /** Total write request packets queue length in #packets */
 113     uint64_t totalWriteQueueSize;
 114
 115     /**
 116      * Bus state used to control the read/write switching and drive
 117      * the scheduling of the next request.
 118      */
 119     BusState busState;
 120
 121     /** bus state for next request event triggered */
 122     BusState busStateNext;
 123
 124     struct MemCtrlStats : public Stats::Group
 125     {
 126         MemCtrlStats(MemCtrl &mc);
 127
 128         void regStats() override;
 129
 130         const MemCtrl &memCtrl;
 131
 132         /** per-requestor average QoS priority */
 133         Stats::VectorStandardDeviation avgPriority;
 134         /**
 135          * per-requestor average QoS distance between assigned and
 136          * queued values
 137          */
 138         Stats::VectorStandardDeviation avgPriorityDistance;
 139
 140         /** per-priority minimum latency */
 141         Stats::Vector priorityMinLatency;
 142         /** per-priority maximum latency */
 143         Stats::Vector priorityMaxLatency;
 144         /** Count the number of turnarounds READ to WRITE */
 145         Stats::Scalar numReadWriteTurnArounds;
 146         /** Count the number of turnarounds WRITE to READ */
 147         Stats::Scalar numWriteReadTurnArounds;
 148         /** Count the number of times bus staying in READ state */
 149         Stats::Scalar numStayReadState;
 150         /** Count the number of times bus staying in WRITE state */
 151         Stats::Scalar numStayWriteState;
 152     } stats;
 153
 154     /** Pointer to the System object */
 155     System* _system;
 156
 157     /**
 158      * Initializes dynamically counters and
 159      * statistics for a given Requestor
 160      *
 161      * @param id the requestor's ID
 162      */
 163     void addRequestor(const RequestorID id);
 164
 165     /**
 166      * Called upon receiving a request or
 167      * updates statistics and updates queues status
 168      *
 169      * @param dir request direction
 170      * @param id requestor id
 171      * @param qos packet qos value
 172      * @param addr packet address
 173      * @param entries number of entries to record
 174      */
 175     void logRequest(BusState dir, RequestorID id, uint8_t qos,
 176                     Addr addr, uint64_t entries);
 177
 178     /**
 179      * Called upon receiving a response,
 180      * updates statistics and updates queues status
 181      *
 182      * @param dir response direction
 183      * @param id requestor id
 184      * @param qos packet qos value
 185      * @param addr packet address
 186      * @param entries number of entries to record
 187      * @param delay response delay
 188      */
 189     void logResponse(BusState dir, RequestorID id, uint8_t qos,
 190                      Addr addr, uint64_t entries, double delay);
 191
 192     /**
 193      * Assign priority to a packet by executing
 194      * the configured QoS policy.
 195      *
 196      * @param queues_ptr list of pointers to packet queues
 197      * @param queue_entry_size size in bytes per each packet in the queue
 198      * @param pkt pointer to the Packet
 199      * @return a QoS priority value
 200      */
 201     template<typename Queues>
 202     uint8_t qosSchedule(std::initializer_list<Queues*> queues_ptr,
 203                         uint64_t queue_entry_size, const PacketPtr pkt);
 204
 205     using SimObject::schedule;
 206     uint8_t schedule(RequestorID id, uint64_t data);
 207     uint8_t schedule(const PacketPtr pkt);
 208
 209     /**
 210      * Returns next bus direction (READ or WRITE)
 211      * based on configured policy.
 212      */
 213     BusState selectNextBusState();
 214
 215     /**
 216      * Set current bus direction (READ or WRITE)
 217      * from next selected one
 218      */
 219     void setCurrentBusState() { busState = busStateNext; }
 220
 221     /**
 222      * Record statistics on turnarounds based on
 223      * busStateNext and busState values
 224      */
 225     void recordTurnaroundStats();
 226
 227     /**
 228      * Escalates/demotes priority of all packets
 229      * belonging to the passed requestor to given
 230      * priority value
 231      *
 232      * @param queues list of pointers to packet queues
 233      * @param queue_entry_size size of an entry in the queue
 234      * @param id requestor whose packets priority will change
 235      * @param tgt_prio target priority value
 236      */
 237     template<typename Queues>
 238     void escalate(std::initializer_list<Queues*> queues,
 239                   uint64_t queue_entry_size,
 240                   RequestorID id, uint8_t tgt_prio);
 241
 242     /**
 243      * Escalates/demotes priority of all packets
 244      * belonging to the passed requestor to given
 245      * priority value in a specified cluster of queues
 246      * (e.g. read queues or write queues) which is passed
 247      * as an argument to the function.
 248      * The curr_prio/tgt_prio parameters are queue selectors in the
 249      * queue cluster.
 250      *
 251      * @param queues reference to packet queues
 252      * @param queue_entry_size size of an entry in the queue
 253      * @param id requestor whose packets priority will change
 254      * @param curr_prio source queue priority value
 255      * @param tgt_prio target queue priority value
 256      */
 257     template<typename Queues>
 258     void escalateQueues(Queues& queues, uint64_t queue_entry_size,
 259                         RequestorID id, uint8_t curr_prio, uint8_t tgt_prio);
 260
 261   public:
 262     /**
 263      * QoS Memory base class
 264      *
 265      * @param p pointer to QoSMemCtrl parameters
 266      */
 267     MemCtrl(const QoSMemCtrlParams*);
 268
 269     virtual ~MemCtrl();
 270
 271     /**
 272      * Gets the current bus state
 273      *
 274      * @return current bus state
 275      */
 276     BusState getBusState() const { return busState; }
 277
 278     /**
 279      * Gets the next bus state
 280      *
 281      * @return next bus state
 282      */
 283     BusState getBusStateNext() const { return busStateNext; }
 284
 285     /**
 286      * hasRequestor returns true if the selected requestor(ID) has
 287      * been registered in the memory controller, which happens if
 288      * the memory controller has received at least a packet from
 289      * that requestor.
 290      *
 291      * @param id requestor id to lookup
 292      * @return true if the memory controller has received a packet
 293      *         from the requestor, false otherwise.
 294      */
 295     bool hasRequestor(RequestorID id) const
 296     {
 297         return requestors.find(id) != requestors.end();
 298     }
 299
 300     /**
 301      * Gets a READ queue size
 302      *
 303      * @param prio QoS Priority of the queue
 304      * @return queue size in packets
 305      */
 306     uint64_t getReadQueueSize(const uint8_t prio) const
 307     { return readQueueSizes[prio]; }
 308
 309     /**
 310      * Gets a WRITE queue size
 311      *
 312      * @param prio QoS Priority of the queue
 313      * @return queue size in packets
 314      */
 315     uint64_t getWriteQueueSize(const uint8_t prio) const
 316     { return writeQueueSizes[prio]; }
 317
 318     /**
 319      * Gets the total combined READ queues size
 320      *
 321      * @return total queues size in packets
 322      */
 323     uint64_t getTotalReadQueueSize() const { return totalReadQueueSize; }
 324
 325     /**
 326      * Gets the total combined WRITE queues size
 327      *
 328      * @return total queues size in packets
 329      */
 330     uint64_t getTotalWriteQueueSize() const { return totalWriteQueueSize; }
 331
 332     /**
 333      * Gets the last service tick related to a QoS Priority
 334      *
 335      * @param prio QoS Priority
 336      * @return tick
 337      */
 338     Tick getServiceTick(const uint8_t prio) const { return serviceTick[prio]; }
 339
 340     /**
 341      * Gets the total number of priority levels in the
 342      * QoS memory controller.
 343      *
 344      * @return total number of priority levels
 345      */
 346     uint8_t numPriorities() const { return _numPriorities; }
 347
 348     /** read the system pointer
 349      * @return pointer to the system object */
 350     System* system() const { return _system; }
 351 };
 352
 353 template<typename Queues>
 354 void
 355 MemCtrl::escalateQueues(Queues& queues, uint64_t queue_entry_size,
 356                         RequestorID id, uint8_t curr_prio, uint8_t tgt_prio)
 357 {
 358     auto it = queues[curr_prio].begin();
 359     while (it != queues[curr_prio].end()) {
 360         // No packets left to move
 361         if (packetPriorities[id][curr_prio] == 0)
 362             break;
 363
 364         auto pkt = *it;
 365
 366         DPRINTF(QOS,
 367                 "QoSMemCtrl::escalate checking priority %d packet "
 368                 "id %d address %d\n", curr_prio,
 369                 pkt->requestorId(), pkt->getAddr());
 370
 371         // Found a packet to move
 372         if (pkt->requestorId() == id) {
 373
 374             uint64_t moved_entries = divCeil(pkt->getSize(),
 375                                              queue_entry_size);
 376
 377             DPRINTF(QOS,
 378                     "QoSMemCtrl::escalate Requestor %s [id %d] moving "
 379                     "packet addr %d size %d (p size %d) from priority %d "
 380                     "to priority %d - "
 381                     "this requestor packets %d (entries to move %d)\n",
 382                     requestors[id], id, pkt->getAddr(),
 383                     pkt->getSize(),
 384                     queue_entry_size, curr_prio, tgt_prio,
 385                     packetPriorities[id][curr_prio], moved_entries);
 386
 387
 388             if (pkt->isRead()) {
 389                 panic_if(readQueueSizes[curr_prio] < moved_entries,
 390                          "QoSMemCtrl::escalate requestor %s negative READ "
 391                          "packets for priority %d",
 392                         requestors[id], tgt_prio);
 393                 readQueueSizes[curr_prio] -= moved_entries;
 394                 readQueueSizes[tgt_prio] += moved_entries;
 395             } else if (pkt->isWrite()) {
 396                 panic_if(writeQueueSizes[curr_prio] < moved_entries,
 397                          "QoSMemCtrl::escalate requestor %s negative WRITE "
 398                          "packets for priority %d",
 399                         requestors[id], tgt_prio);
 400                 writeQueueSizes[curr_prio] -= moved_entries;
 401                 writeQueueSizes[tgt_prio] += moved_entries;
 402             }
 403
 404             // Change QoS priority and move packet
 405             pkt->qosValue(tgt_prio);
 406             queues[tgt_prio].push_back(pkt);
 407
 408             // Erase element from source packet queue, this will
 409             // increment the iterator
 410             it = queues[curr_prio].erase(it);
 411             panic_if(packetPriorities[id][curr_prio] < moved_entries,
 412                      "QoSMemCtrl::escalate requestor %s negative packets "
 413                      "for priority %d",
 414                      requestors[id], tgt_prio);
 415
 416             packetPriorities[id][curr_prio] -= moved_entries;
 417             packetPriorities[id][tgt_prio] += moved_entries;
 418         } else {
 419             // Increment iterator to next location in the queue
 420             it++;
 421         }
 422     }
 423 }
 424
 425 template<typename Queues>
 426 void
 427 MemCtrl::escalate(std::initializer_list<Queues*> queues,
 428                   uint64_t queue_entry_size,
 429                   RequestorID id, uint8_t tgt_prio)
 430 {
 431     // If needed, initialize all counters and statistics
 432     // for this requestor
 433     addRequestor(id);
 434
 435     DPRINTF(QOS,
 436             "QoSMemCtrl::escalate Requestor %s [id %d] to priority "
 437             "%d (currently %d packets)\n",requestors[id], id, tgt_prio,
 438             packetPriorities[id][tgt_prio]);
 439
 440     for (uint8_t curr_prio = 0; curr_prio < numPriorities(); ++curr_prio) {
 441         // Skip target priority
 442         if (curr_prio == tgt_prio)
 443             continue;
 444
 445         // Process other priority packet
 446         while (packetPriorities[id][curr_prio] > 0) {
 447             DPRINTF(QOS,
 448                     "QoSMemCtrl::escalate MID %d checking priority %d "
 449                     "(packets %d)- current packets in prio %d:  %d\n"
 450                     "\t(source read %d source write %d target read %d, "
 451                     "target write %d)\n",
 452                     id, curr_prio, packetPriorities[id][curr_prio],
 453                     tgt_prio, packetPriorities[id][tgt_prio],
 454                     readQueueSizes[curr_prio],
 455                     writeQueueSizes[curr_prio], readQueueSizes[tgt_prio],
 456                     writeQueueSizes[tgt_prio]);
 457
 458             // Check both read and write queue
 459             for (auto q : queues) {
 460                 escalateQueues(*q, queue_entry_size, id,
 461                                curr_prio, tgt_prio);
 462             }
 463         }
 464     }
 465
 466     DPRINTF(QOS,
 467             "QoSMemCtrl::escalate Completed requestor %s [id %d] to priority "
 468             "%d (now %d packets)\n\t(total read %d, total write %d)\n",
 469             requestors[id], id, tgt_prio, packetPriorities[id][tgt_prio],
 470             readQueueSizes[tgt_prio], writeQueueSizes[tgt_prio]);
 471 }
 472
 473 template<typename Queues>
 474 uint8_t
 475 MemCtrl::qosSchedule(std::initializer_list<Queues*> queues,
 476                      const uint64_t queue_entry_size,
 477                      const PacketPtr pkt)
 478 {
 479     // Schedule packet.
 480     uint8_t pkt_priority = schedule(pkt);
 481
 482     assert(pkt_priority < numPriorities());
 483
 484     pkt->qosValue(pkt_priority);
 485
 486     if (qosSyncroScheduler) {
 487         // Call the scheduling function on all other requestors.
 488         for (const auto& requestor : requestors) {
 489
 490             if (requestor.first == pkt->requestorId())
 491                 continue;
 492
 493             uint8_t prio = schedule(requestor.first, 0);
 494
 495             if (qosPriorityEscalation) {
 496                 DPRINTF(QOS,
 497                         "QoSMemCtrl::qosSchedule: (syncro) escalating "
 498                         "REQUESTOR %s to assigned priority %d\n",
 499                         _system->getRequestorName(requestor.first),
 500                         prio);
 501                 escalate(queues, queue_entry_size, requestor.first, prio);
 502             }
 503         }
 504     }
 505
 506     if (qosPriorityEscalation) {
 507         DPRINTF(QOS,
 508                 "QoSMemCtrl::qosSchedule: escalating "
 509                 "REQUESTOR %s to assigned priority %d\n",
 510                 _system->getRequestorName(pkt->requestorId()),
 511                 pkt_priority);
 512         escalate(queues, queue_entry_size, pkt->requestorId(), pkt_priority);
 513     }
 514
 515     // Update last service tick for selected priority
 516     serviceTick[pkt_priority] = curTick();
 517
 518     return pkt_priority;
 519 }
 520
 521 } // namespace QoS
 522
 523 #endif /* __MEM_QOS_MEM_CTRL_HH__ */