src/dev/arm/gic_v3_its.cc

   1 /*
   2  * Copyright (c) 2019 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  *
  37  * Authors: Giacomo Travaglini
  38  */
  39
  40 #include "dev/arm/gic_v3_its.hh"
  41
  42 #include "debug/AddrRanges.hh"
  43 #include "debug/Drain.hh"
  44 #include "debug/GIC.hh"
  45 #include "debug/ITS.hh"
  46 #include "dev/arm/gic_v3.hh"
  47 #include "dev/arm/gic_v3_distributor.hh"
  48 #include "dev/arm/gic_v3_redistributor.hh"
  49 #include "mem/packet_access.hh"
  50
  51 #define COMMAND(x, method) { x, DispatchEntry(#x, method) }
  52
  53 const AddrRange Gicv3Its::GITS_BASER(0x0100, 0x0138);
  54
  55 const uint32_t Gicv3Its::CTLR_QUIESCENT = 0x80000000;
  56
  57 ItsProcess::ItsProcess(Gicv3Its &_its)
  58   : its(_its), coroutine(nullptr)
  59 {
  60 }
  61
  62 ItsProcess::~ItsProcess()
  63 {
  64 }
  65
  66 void
  67 ItsProcess::reinit()
  68 {
  69     coroutine.reset(new Coroutine(
  70         std::bind(&ItsProcess::main, this, std::placeholders::_1)));
  71 }
  72
  73 const std::string
  74 ItsProcess::name() const
  75 {
  76     return its.name();
  77 }
  78
  79 ItsAction
  80 ItsProcess::run(PacketPtr pkt)
  81 {
  82     assert(coroutine != nullptr);
  83     assert(*coroutine);
  84     return (*coroutine)(pkt).get();
  85 }
  86
  87 void
  88 ItsProcess::doRead(Yield &yield, Addr addr, void *ptr, size_t size)
  89 {
  90     ItsAction a;
  91     a.type = ItsActionType::SEND_REQ;
  92
  93     RequestPtr req = std::make_shared<Request>(
  94         addr, size, 0, its.masterId);
  95
  96     req->taskId(ContextSwitchTaskId::DMA);
  97
  98     a.pkt = new Packet(req, MemCmd::ReadReq);
  99     a.pkt->dataStatic(ptr);
 100
 101     a.delay = 0;
 102
 103     PacketPtr pkt = yield(a).get();
 104
 105     assert(pkt);
 106     assert(pkt->getSize() >= size);
 107
 108     delete pkt;
 109 }
 110
 111 void
 112 ItsProcess::doWrite(Yield &yield, Addr addr, void *ptr, size_t size)
 113 {
 114     ItsAction a;
 115     a.type = ItsActionType::SEND_REQ;
 116
 117     RequestPtr req = std::make_shared<Request>(
 118         addr, size, 0, its.masterId);
 119
 120     req->taskId(ContextSwitchTaskId::DMA);
 121
 122     a.pkt = new Packet(req, MemCmd::WriteReq);
 123     a.pkt->dataStatic(ptr);
 124
 125     a.delay = 0;
 126
 127     PacketPtr pkt = yield(a).get();
 128
 129     assert(pkt);
 130     assert(pkt->getSize() >= size);
 131
 132     delete pkt;
 133 }
 134
 135 void
 136 ItsProcess::terminate(Yield &yield)
 137 {
 138     ItsAction a;
 139     a.type = ItsActionType::TERMINATE;
 140     a.pkt = NULL;
 141     a.delay = 0;
 142     yield(a);
 143 }
 144
 145 void
 146 ItsProcess::writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte)
 147 {
 148     const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);
 149     const Addr address = base + device_id;
 150
 151     DPRINTF(ITS, "Writing DTE at address %#x: %#x\n", address, dte);
 152
 153     doWrite(yield, address, &dte, sizeof(dte));
 154 }
 155
 156 void
 157 ItsProcess::writeIrqTranslationTable(
 158     Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte)
 159 {
 160     const Addr address = itt_base + event_id;
 161
 162     doWrite(yield, address, &itte, sizeof(itte));
 163
 164     DPRINTF(ITS, "Writing ITTE at address %#x: %#x\n", address, itte);
 165 }
 166
 167 void
 168 ItsProcess::writeIrqCollectionTable(
 169     Yield &yield, uint32_t collection_id, CTE cte)
 170 {
 171     const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);
 172     const Addr address = base + collection_id;
 173
 174     doWrite(yield, address, &cte, sizeof(cte));
 175
 176     DPRINTF(ITS, "Writing CTE at address %#x: %#x\n", address, cte);
 177 }
 178
 179 uint64_t
 180 ItsProcess::readDeviceTable(Yield &yield, uint32_t device_id)
 181 {
 182     const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);
 183     const Addr address = base + device_id;
 184
 185     uint64_t dte;
 186     doRead(yield, address, &dte, sizeof(dte));
 187
 188     DPRINTF(ITS, "Reading DTE at address %#x: %#x\n", address, dte);
 189     return dte;
 190 }
 191
 192 uint64_t
 193 ItsProcess::readIrqTranslationTable(
 194     Yield &yield, const Addr itt_base, uint32_t event_id)
 195 {
 196     const Addr address = itt_base + event_id;
 197
 198     uint64_t itte;
 199     doRead(yield, address, &itte, sizeof(itte));
 200
 201     DPRINTF(ITS, "Reading ITTE at address %#x: %#x\n", address, itte);
 202     return itte;
 203 }
 204
 205 uint64_t
 206 ItsProcess::readIrqCollectionTable(Yield &yield, uint32_t collection_id)
 207 {
 208     const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);
 209     const Addr address = base + collection_id;
 210
 211     uint64_t cte;
 212     doRead(yield, address, &cte, sizeof(cte));
 213
 214     DPRINTF(ITS, "Reading CTE at address %#x: %#x\n", address, cte);
 215     return cte;
 216 }
 217
 218 ItsTranslation::ItsTranslation(Gicv3Its &_its)
 219   : ItsProcess(_its)
 220 {
 221     reinit();
 222     its.pendingTranslations++;
 223     its.gitsControl.quiescent = 0;
 224 }
 225
 226 ItsTranslation::~ItsTranslation()
 227 {
 228     assert(its.pendingTranslations >= 1);
 229     its.pendingTranslations--;
 230     if (!its.pendingTranslations && !its.pendingCommands)
 231         its.gitsControl.quiescent = 1;
 232 }
 233
 234 void
 235 ItsTranslation::main(Yield &yield)
 236 {
 237     PacketPtr pkt = yield.get();
 238
 239     const uint32_t device_id = pkt->req->streamId();
 240     const uint32_t event_id = pkt->getLE<uint32_t>();
 241
 242     auto result = translateLPI(yield, device_id, event_id);
 243
 244     uint32_t intid = result.first;
 245     Gicv3Redistributor *redist = result.second;
 246
 247     // Set the LPI in the redistributor
 248     redist->setClrLPI(intid, true);
 249
 250     // Update the value in GITS_TRANSLATER only once we know
 251     // there was no error in the tranlation process (before
 252     // terminating the translation
 253     its.gitsTranslater = event_id;
 254
 255     terminate(yield);
 256 }
 257
 258 std::pair<uint32_t, Gicv3Redistributor *>
 259 ItsTranslation::translateLPI(Yield &yield, uint32_t device_id,
 260                              uint32_t event_id)
 261 {
 262     if (its.deviceOutOfRange(device_id)) {
 263         terminate(yield);
 264     }
 265
 266     DTE dte = readDeviceTable(yield, device_id);
 267
 268     if (!dte.valid || its.idOutOfRange(event_id, dte.ittRange)) {
 269         terminate(yield);
 270     }
 271
 272     ITTE itte = readIrqTranslationTable(yield, dte.ittAddress, event_id);
 273     const auto collection_id = itte.icid;
 274
 275     if (!itte.valid || its.collectionOutOfRange(collection_id)) {
 276         terminate(yield);
 277     }
 278
 279     CTE cte = readIrqCollectionTable(yield, collection_id);
 280
 281     if (!cte.valid) {
 282         terminate(yield);
 283     }
 284
 285     // Returning the INTID and the target Redistributor
 286     return std::make_pair(itte.intNum, its.getRedistributor(cte));
 287 }
 288
 289 ItsCommand::DispatchTable ItsCommand::cmdDispatcher =
 290 {
 291     COMMAND(CLEAR, &ItsCommand::clear),
 292     COMMAND(DISCARD, &ItsCommand::discard),
 293     COMMAND(INT, &ItsCommand::doInt),
 294     COMMAND(INV, &ItsCommand::inv),
 295     COMMAND(INVALL, &ItsCommand::invall),
 296     COMMAND(MAPC, &ItsCommand::mapc),
 297     COMMAND(MAPD, &ItsCommand::mapd),
 298     COMMAND(MAPI, &ItsCommand::mapi),
 299     COMMAND(MAPTI, &ItsCommand::mapti),
 300     COMMAND(MOVALL, &ItsCommand::movall),
 301     COMMAND(MOVI, &ItsCommand::movi),
 302     COMMAND(SYNC, &ItsCommand::sync),
 303     COMMAND(VINVALL, &ItsCommand::vinvall),
 304     COMMAND(VMAPI, &ItsCommand::vmapi),
 305     COMMAND(VMAPP, &ItsCommand::vmapp),
 306     COMMAND(VMAPTI, &ItsCommand::vmapti),
 307     COMMAND(VMOVI, &ItsCommand::vmovi),
 308     COMMAND(VMOVP, &ItsCommand::vmovp),
 309     COMMAND(VSYNC, &ItsCommand::vsync),
 310 };
 311
 312 ItsCommand::ItsCommand(Gicv3Its &_its)
 313   : ItsProcess(_its)
 314 {
 315     reinit();
 316     its.pendingCommands = true;
 317
 318     its.gitsControl.quiescent = 0;
 319 }
 320
 321 ItsCommand::~ItsCommand()
 322 {
 323     its.pendingCommands = false;
 324
 325     if (!its.pendingTranslations)
 326         its.gitsControl.quiescent = 1;
 327 }
 328
 329 std::string
 330 ItsCommand::commandName(uint32_t cmd)
 331 {
 332     const auto entry = cmdDispatcher.find(cmd);
 333     return entry != cmdDispatcher.end() ? entry->second.name : "INVALID";
 334 }
 335
 336 void
 337 ItsCommand::main(Yield &yield)
 338 {
 339     ItsAction a;
 340     a.type = ItsActionType::INITIAL_NOP;
 341     a.pkt = nullptr;
 342     a.delay = 0;
 343     yield(a);
 344
 345     while (its.gitsCwriter.offset != its.gitsCreadr.offset) {
 346         CommandEntry command;
 347
 348         // Reading the command from CMDQ
 349         readCommand(yield, command);
 350
 351         processCommand(yield, command);
 352
 353         its.incrementReadPointer();
 354     }
 355
 356     terminate(yield);
 357 }
 358
 359 void
 360 ItsCommand::readCommand(Yield &yield, CommandEntry &command)
 361 {
 362     // read the command pointed by GITS_CREADR
 363     const Addr cmd_addr =
 364         (its.gitsCbaser.physAddr << 12) + (its.gitsCreadr.offset << 5);
 365
 366     doRead(yield, cmd_addr, &command, sizeof(command));
 367
 368     DPRINTF(ITS, "Command %s read from queue at address: %#x\n",
 369             commandName(command.type), cmd_addr);
 370     DPRINTF(ITS, "dw0: %#x dw1: %#x dw2: %#x dw3: %#x\n",
 371             command.raw[0], command.raw[1], command.raw[2], command.raw[3]);
 372 }
 373
 374 void
 375 ItsCommand::processCommand(Yield &yield, CommandEntry &command)
 376 {
 377     const auto entry = cmdDispatcher.find(command.type);
 378
 379     if (entry != cmdDispatcher.end()) {
 380         // Execute the command
 381         entry->second.exec(this, yield, command);
 382     } else {
 383         panic("Unrecognized command type: %u", command.type);
 384     }
 385 }
 386
 387 void
 388 ItsCommand::clear(Yield &yield, CommandEntry &command)
 389 {
 390     if (deviceOutOfRange(command)) {
 391         its.incrementReadPointer();
 392         terminate(yield);
 393     }
 394
 395     DTE dte = readDeviceTable(yield, command.deviceId);
 396
 397     if (!dte.valid || idOutOfRange(command, dte)) {
 398         its.incrementReadPointer();
 399         terminate(yield);
 400     }
 401
 402     ITTE itte = readIrqTranslationTable(
 403         yield, dte.ittAddress, command.eventId);
 404
 405     if (!itte.valid) {
 406         its.incrementReadPointer();
 407         terminate(yield);
 408     }
 409
 410     const auto collection_id = itte.icid;
 411     CTE cte = readIrqCollectionTable(yield, collection_id);
 412
 413     if (!cte.valid) {
 414         its.incrementReadPointer();
 415         terminate(yield);
 416     }
 417
 418     // Clear the LPI in the redistributor
 419     its.getRedistributor(cte)->setClrLPI(itte.intNum, false);
 420 }
 421
 422 void
 423 ItsCommand::discard(Yield &yield, CommandEntry &command)
 424 {
 425     if (deviceOutOfRange(command)) {
 426         its.incrementReadPointer();
 427         terminate(yield);
 428     }
 429
 430     DTE dte = readDeviceTable(yield, command.deviceId);
 431
 432     if (!dte.valid || idOutOfRange(command, dte)) {
 433         its.incrementReadPointer();
 434         terminate(yield);
 435     }
 436
 437     ITTE itte = readIrqTranslationTable(
 438         yield, dte.ittAddress, command.eventId);
 439
 440     if (!itte.valid) {
 441         its.incrementReadPointer();
 442         terminate(yield);
 443     }
 444
 445     const auto collection_id = itte.icid;
 446     Gicv3Its::CTE cte = readIrqCollectionTable(yield, collection_id);
 447
 448     if (!cte.valid) {
 449         its.incrementReadPointer();
 450         terminate(yield);
 451     }
 452
 453     its.getRedistributor(cte)->setClrLPI(itte.intNum, false);
 454
 455     // Then removes the mapping from the ITT (invalidating)
 456     itte.valid = 0;
 457     writeIrqTranslationTable(
 458         yield, dte.ittAddress, command.eventId, itte);
 459 }
 460
 461 void
 462 ItsCommand::doInt(Yield &yield, CommandEntry &command)
 463 {
 464     if (deviceOutOfRange(command)) {
 465         its.incrementReadPointer();
 466         terminate(yield);
 467     }
 468
 469     DTE dte = readDeviceTable(yield, command.deviceId);
 470
 471     if (!dte.valid || idOutOfRange(command, dte)) {
 472         its.incrementReadPointer();
 473         terminate(yield);
 474     }
 475
 476     ITTE itte = readIrqTranslationTable(
 477         yield, dte.ittAddress, command.eventId);
 478
 479     if (!itte.valid) {
 480         its.incrementReadPointer();
 481         terminate(yield);
 482     }
 483
 484     const auto collection_id = itte.icid;
 485     CTE cte = readIrqCollectionTable(yield, collection_id);
 486
 487     if (!cte.valid) {
 488         its.incrementReadPointer();
 489         terminate(yield);
 490     }
 491
 492     // Set the LPI in the redistributor
 493     its.getRedistributor(cte)->setClrLPI(itte.intNum, true);
 494 }
 495
 496 void
 497 ItsCommand::inv(Yield &yield, CommandEntry &command)
 498 {
 499     if (deviceOutOfRange(command)) {
 500         its.incrementReadPointer();
 501         terminate(yield);
 502     }
 503
 504     DTE dte = readDeviceTable(yield, command.deviceId);
 505
 506     if (!dte.valid || idOutOfRange(command, dte)) {
 507         its.incrementReadPointer();
 508         terminate(yield);
 509     }
 510
 511     ITTE itte = readIrqTranslationTable(
 512         yield, dte.ittAddress, command.eventId);
 513
 514     if (!itte.valid) {
 515         its.incrementReadPointer();
 516         terminate(yield);
 517     }
 518
 519     const auto collection_id = itte.icid;
 520     CTE cte = readIrqCollectionTable(yield, collection_id);
 521
 522     if (!cte.valid) {
 523         its.incrementReadPointer();
 524         terminate(yield);
 525     }
 526     // Do nothing since caching is currently not supported in
 527     // Redistributor
 528 }
 529
 530 void
 531 ItsCommand::invall(Yield &yield, CommandEntry &command)
 532 {
 533     if (collectionOutOfRange(command)) {
 534         its.incrementReadPointer();
 535         terminate(yield);
 536     }
 537
 538     const auto icid = bits(command.raw[2], 15, 0);
 539
 540     CTE cte = readIrqCollectionTable(yield, icid);
 541
 542     if (!cte.valid) {
 543         its.incrementReadPointer();
 544         terminate(yield);
 545     }
 546     // Do nothing since caching is currently not supported in
 547     // Redistributor
 548 }
 549
 550 void
 551 ItsCommand::mapc(Yield &yield, CommandEntry &command)
 552 {
 553     if (collectionOutOfRange(command)) {
 554         its.incrementReadPointer();
 555         terminate(yield);
 556     }
 557
 558     CTE cte = 0;
 559     cte.valid = bits(command.raw[2], 63);
 560     cte.rdBase = bits(command.raw[2], 50, 16);
 561
 562     const auto icid = bits(command.raw[2], 15, 0);
 563
 564     writeIrqCollectionTable(yield, icid, cte);
 565 }
 566
 567 void
 568 ItsCommand::mapd(Yield &yield, CommandEntry &command)
 569 {
 570     if (deviceOutOfRange(command) || sizeOutOfRange(command)) {
 571         its.incrementReadPointer();
 572         terminate(yield);
 573     }
 574
 575     DTE dte = 0;
 576     dte.valid = bits(command.raw[2], 63);
 577     dte.ittAddress = mbits(command.raw[2], 51, 8);
 578     dte.ittRange = bits(command.raw[1], 4, 0);
 579
 580     writeDeviceTable(yield, command.deviceId, dte);
 581 }
 582
 583 void
 584 ItsCommand::mapi(Yield &yield, CommandEntry &command)
 585 {
 586     if (deviceOutOfRange(command)) {
 587         its.incrementReadPointer();
 588         terminate(yield);
 589     }
 590
 591     if (collectionOutOfRange(command)) {
 592         its.incrementReadPointer();
 593         terminate(yield);
 594     }
 595
 596     DTE dte = readDeviceTable(yield, command.deviceId);
 597
 598     if (!dte.valid || idOutOfRange(command, dte) ||
 599         its.lpiOutOfRange(command.eventId)) {
 600
 601         its.incrementReadPointer();
 602         terminate(yield);
 603     }
 604
 605     Gicv3Its::ITTE itte = readIrqTranslationTable(
 606         yield, dte.ittAddress, command.eventId);
 607
 608     itte.valid = 1;
 609     itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;
 610     itte.intNum = command.eventId;
 611     itte.icid = bits(command.raw[2], 15, 0);
 612
 613     writeIrqTranslationTable(
 614         yield, dte.ittAddress, command.eventId, itte);
 615 }
 616
 617 void
 618 ItsCommand::mapti(Yield &yield, CommandEntry &command)
 619 {
 620     if (deviceOutOfRange(command)) {
 621         its.incrementReadPointer();
 622         terminate(yield);
 623     }
 624
 625     if (collectionOutOfRange(command)) {
 626         its.incrementReadPointer();
 627         terminate(yield);
 628     }
 629
 630     DTE dte = readDeviceTable(yield, command.deviceId);
 631
 632     const auto pintid = bits(command.raw[1], 63, 32);
 633
 634     if (!dte.valid || idOutOfRange(command, dte) ||
 635         its.lpiOutOfRange(pintid)) {
 636
 637         its.incrementReadPointer();
 638         terminate(yield);
 639     }
 640
 641     ITTE itte = readIrqTranslationTable(
 642         yield, dte.ittAddress, command.eventId);
 643
 644     itte.valid = 1;
 645     itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;
 646     itte.intNum = pintid;
 647     itte.icid = bits(command.raw[2], 15, 0);
 648
 649     writeIrqTranslationTable(
 650         yield, dte.ittAddress, command.eventId, itte);
 651 }
 652
 653 void
 654 ItsCommand::movall(Yield &yield, CommandEntry &command)
 655 {
 656     const uint64_t rd1 = bits(command.raw[2], 50, 16);
 657     const uint64_t rd2 = bits(command.raw[3], 50, 16);
 658
 659     if (rd1 != rd2) {
 660         Gicv3Redistributor * redist1 = its.getRedistributor(rd1);
 661         Gicv3Redistributor * redist2 = its.getRedistributor(rd2);
 662
 663         its.moveAllPendingState(redist1, redist2);
 664     }
 665 }
 666
 667 void
 668 ItsCommand::movi(Yield &yield, CommandEntry &command)
 669 {
 670     if (deviceOutOfRange(command)) {
 671         its.incrementReadPointer();
 672         terminate(yield);
 673     }
 674
 675     if (collectionOutOfRange(command)) {
 676         its.incrementReadPointer();
 677         terminate(yield);
 678     }
 679
 680     DTE dte = readDeviceTable(yield, command.deviceId);
 681
 682     if (!dte.valid || idOutOfRange(command, dte)) {
 683         its.incrementReadPointer();
 684         terminate(yield);
 685     }
 686
 687     ITTE itte = readIrqTranslationTable(
 688         yield, dte.ittAddress, command.eventId);
 689
 690     if (!itte.valid || itte.intType == Gicv3Its::VIRTUAL_INTERRUPT) {
 691         its.incrementReadPointer();
 692         terminate(yield);
 693     }
 694
 695     const auto collection_id1 = itte.icid;
 696     CTE cte1 = readIrqCollectionTable(yield, collection_id1);
 697
 698     if (!cte1.valid) {
 699         its.incrementReadPointer();
 700         terminate(yield);
 701     }
 702
 703     const auto collection_id2 = bits(command.raw[2], 15, 0);
 704     CTE cte2 = readIrqCollectionTable(yield, collection_id2);
 705
 706     if (!cte2.valid) {
 707         its.incrementReadPointer();
 708         terminate(yield);
 709     }
 710
 711     Gicv3Redistributor *first_redist = its.getRedistributor(cte1);
 712     Gicv3Redistributor *second_redist = its.getRedistributor(cte2);
 713
 714     if (second_redist != first_redist) {
 715         // move pending state of the interrupt from one redistributor
 716         // to the other.
 717         if (first_redist->isPendingLPI(itte.intNum)) {
 718             first_redist->setClrLPI(itte.intNum, false);
 719             second_redist->setClrLPI(itte.intNum, true);
 720         }
 721     }
 722
 723     itte.icid = collection_id2;
 724     writeIrqTranslationTable(
 725         yield, dte.ittAddress, command.eventId, itte);
 726 }
 727
 728 void
 729 ItsCommand::sync(Yield &yield, CommandEntry &command)
 730 {
 731     warn("ITS %s command unimplemented", __func__);
 732 }
 733
 734 void
 735 ItsCommand::vinvall(Yield &yield, CommandEntry &command)
 736 {
 737     panic("ITS %s command unimplemented", __func__);
 738 }
 739
 740 void
 741 ItsCommand::vmapi(Yield &yield, CommandEntry &command)
 742 {
 743     panic("ITS %s command unimplemented", __func__);
 744 }
 745
 746 void
 747 ItsCommand::vmapp(Yield &yield, CommandEntry &command)
 748 {
 749     panic("ITS %s command unimplemented", __func__);
 750 }
 751
 752 void
 753 ItsCommand::vmapti(Yield &yield, CommandEntry &command)
 754 {
 755     panic("ITS %s command unimplemented", __func__);
 756 }
 757
 758 void
 759 ItsCommand::vmovi(Yield &yield, CommandEntry &command)
 760 {
 761     panic("ITS %s command unimplemented", __func__);
 762 }
 763
 764 void
 765 ItsCommand::vmovp(Yield &yield, CommandEntry &command)
 766 {
 767     panic("ITS %s command unimplemented", __func__);
 768 }
 769
 770 void
 771 ItsCommand::vsync(Yield &yield, CommandEntry &command)
 772 {
 773     panic("ITS %s command unimplemented", __func__);
 774 }
 775
 776 Gicv3Its::Gicv3Its(const Gicv3ItsParams *params)
 777  : BasicPioDevice(params, params->pio_size),
 778    dmaPort(name() + ".dma", *this),
 779    gitsControl(CTLR_QUIESCENT),
 780    gitsTyper(params->gits_typer),
 781    gitsCbaser(0), gitsCreadr(0),
 782    gitsCwriter(0), gitsIidr(0),
 783    tableBases(NUM_BASER_REGS, 0),
 784    masterId(params->system->getMasterId(this)),
 785    gic(nullptr),
 786    commandEvent([this] { checkCommandQueue(); }, name()),
 787    pendingCommands(false),
 788    pendingTranslations(0)
 789 {
 790     BASER device_baser = 0;
 791     device_baser.type = DEVICE_TABLE;
 792     device_baser.entrySize = sizeof(uint64_t) - 1;
 793     tableBases[0] = device_baser;
 794
 795     BASER icollect_baser = 0;
 796     icollect_baser.type = COLLECTION_TABLE;
 797     icollect_baser.entrySize = sizeof(uint64_t) - 1;
 798     tableBases[1] = icollect_baser;
 799 }
 800
 801 void
 802 Gicv3Its::setGIC(Gicv3 *_gic)
 803 {
 804     assert(!gic);
 805     gic = _gic;
 806 }
 807
 808 AddrRangeList
 809 Gicv3Its::getAddrRanges() const
 810 {
 811     assert(pioSize != 0);
 812     AddrRangeList ranges;
 813     DPRINTF(AddrRanges, "registering range: %#x-%#x\n", pioAddr, pioSize);
 814     ranges.push_back(RangeSize(pioAddr, pioSize));
 815     return ranges;
 816 }
 817
 818 Tick
 819 Gicv3Its::read(PacketPtr pkt)
 820 {
 821     const Addr addr = pkt->getAddr() - pioAddr;
 822     uint64_t value = 0;
 823
 824     DPRINTF(GIC, "%s register at addr: %#x\n", __func__, addr);
 825
 826     switch (addr) {
 827       case GITS_CTLR:
 828         value = gitsControl;
 829         break;
 830
 831       case GITS_IIDR:
 832         value = gitsIidr;
 833         break;
 834
 835       case GITS_TYPER:
 836         value = gitsTyper;
 837         break;
 838
 839       case GITS_CBASER:
 840         value = gitsCbaser;
 841         break;
 842
 843       case GITS_CBASER + 4:
 844         value = gitsCbaser.high;
 845         break;
 846
 847       case GITS_CWRITER:
 848         value = gitsCwriter;
 849         break;
 850
 851       case GITS_CWRITER + 4:
 852         value = gitsCwriter.high;
 853         break;
 854
 855       case GITS_CREADR:
 856         value = gitsCreadr;
 857         break;
 858
 859       case GITS_CREADR + 4:
 860         value = gitsCreadr.high;
 861         break;
 862
 863       case GITS_PIDR2:
 864         value = gic->getDistributor()->gicdPidr2;
 865         break;
 866
 867       case GITS_TRANSLATER:
 868         value = gitsTranslater;
 869         break;
 870
 871       default:
 872         if (GITS_BASER.contains(addr)) {
 873             auto relative_addr = addr - GITS_BASER.start();
 874             auto baser_index = relative_addr / sizeof(uint64_t);
 875
 876             value = tableBases[baser_index];
 877             break;
 878         } else {
 879             panic("Unrecognized register access\n");
 880         }
 881     }
 882
 883     pkt->setUintX(value, LittleEndianByteOrder);
 884     pkt->makeAtomicResponse();
 885     return pioDelay;
 886 }
 887
 888 Tick
 889 Gicv3Its::write(PacketPtr pkt)
 890 {
 891     Addr addr = pkt->getAddr() - pioAddr;
 892
 893     DPRINTF(GIC, "%s register at addr: %#x\n", __func__, addr);
 894
 895     switch (addr) {
 896       case GITS_CTLR:
 897         assert(pkt->getSize() == sizeof(uint32_t));
 898         gitsControl = (pkt->getLE<uint32_t>() & ~CTLR_QUIESCENT);
 899         // We should check here if the ITS has been disabled, and if
 900         // that's the case, flush GICv3 caches to external memory.
 901         // This is not happening now, since LPI caching is not
 902         // currently implemented in gem5.
 903         break;
 904
 905       case GITS_IIDR:
 906         panic("GITS_IIDR is Read Only\n");
 907
 908       case GITS_TYPER:
 909         panic("GITS_TYPER is Read Only\n");
 910
 911       case GITS_CBASER:
 912         if (pkt->getSize() == sizeof(uint32_t)) {
 913             gitsCbaser.low = pkt->getLE<uint32_t>();
 914         } else {
 915             assert(pkt->getSize() == sizeof(uint64_t));
 916             gitsCbaser = pkt->getLE<uint64_t>();
 917         }
 918
 919         gitsCreadr = 0; // Cleared when CBASER gets written
 920
 921         checkCommandQueue();
 922         break;
 923
 924       case GITS_CBASER + 4:
 925         assert(pkt->getSize() == sizeof(uint32_t));
 926         gitsCbaser.high = pkt->getLE<uint32_t>();
 927
 928         gitsCreadr = 0; // Cleared when CBASER gets written
 929
 930         checkCommandQueue();
 931         break;
 932
 933       case GITS_CWRITER:
 934         if (pkt->getSize() == sizeof(uint32_t)) {
 935             gitsCwriter.low = pkt->getLE<uint32_t>();
 936         } else {
 937             assert(pkt->getSize() == sizeof(uint64_t));
 938             gitsCwriter = pkt->getLE<uint64_t>();
 939         }
 940
 941         checkCommandQueue();
 942         break;
 943
 944       case GITS_CWRITER + 4:
 945         assert(pkt->getSize() == sizeof(uint32_t));
 946         gitsCwriter.high = pkt->getLE<uint32_t>();
 947
 948         checkCommandQueue();
 949         break;
 950
 951       case GITS_CREADR:
 952         panic("GITS_READR is Read Only\n");
 953
 954       case GITS_TRANSLATER:
 955         if (gitsControl.enabled) {
 956             translate(pkt);
 957         }
 958         break;
 959
 960       default:
 961         if (GITS_BASER.contains(addr)) {
 962             auto relative_addr = addr - GITS_BASER.start();
 963             auto baser_index = relative_addr / sizeof(uint64_t);
 964
 965             const uint64_t table_base = tableBases[baser_index];
 966             const uint64_t w_mask = tableBases[baser_index].type ?
 967                 BASER_WMASK : BASER_WMASK_UNIMPL;
 968             const uint64_t val = pkt->getLE<uint64_t>() & w_mask;
 969
 970             tableBases[baser_index] = table_base | val;
 971             break;
 972         } else {
 973             panic("Unrecognized register access\n");
 974         }
 975     }
 976
 977     pkt->makeAtomicResponse();
 978     return pioDelay;
 979 }
 980
 981 bool
 982 Gicv3Its::idOutOfRange(uint32_t event_id, uint8_t itt_range) const
 983 {
 984     const uint32_t id_bits = gitsTyper.idBits;
 985     return event_id >= (1ULL << (id_bits + 1)) ||
 986         event_id >= ((1ULL << itt_range) + 1);
 987 }
 988
 989 bool
 990 Gicv3Its::deviceOutOfRange(uint32_t device_id) const
 991 {
 992     return device_id >= (1ULL << (gitsTyper.devBits + 1));
 993 }
 994
 995 bool
 996 Gicv3Its::sizeOutOfRange(uint32_t size) const
 997 {
 998     return size > gitsTyper.idBits;
 999 }
1000
1001 bool
1002 Gicv3Its::collectionOutOfRange(uint32_t collection_id) const
1003 {
1004     // If GITS_TYPER.CIL == 0, ITS supports 16-bit CollectionID
1005     // Otherwise, #bits is specified by GITS_TYPER.CIDbits
1006     const auto cid_bits = gitsTyper.cil == 0 ?
1007         16 : gitsTyper.cidBits + 1;
1008
1009     return collection_id >= (1ULL << cid_bits);
1010 }
1011
1012 bool
1013 Gicv3Its::lpiOutOfRange(uint32_t intid) const
1014 {
1015     return intid >= (1ULL << (Gicv3Distributor::IDBITS + 1)) ||
1016            (intid < Gicv3Redistributor::SMALLEST_LPI_ID &&
1017             intid != Gicv3::INTID_SPURIOUS);
1018 }
1019
1020 DrainState
1021 Gicv3Its::drain()
1022 {
1023     if (!pendingCommands && !pendingTranslations) {
1024         return DrainState::Drained;
1025     } else {
1026         DPRINTF(Drain, "GICv3 ITS not drained\n");
1027         return DrainState::Draining;
1028     }
1029 }
1030
1031 void
1032 Gicv3Its::serialize(CheckpointOut & cp) const
1033 {
1034     SERIALIZE_SCALAR(gitsControl);
1035     SERIALIZE_SCALAR(gitsTyper);
1036     SERIALIZE_SCALAR(gitsCbaser);
1037     SERIALIZE_SCALAR(gitsCreadr);
1038     SERIALIZE_SCALAR(gitsCwriter);
1039     SERIALIZE_SCALAR(gitsIidr);
1040
1041     SERIALIZE_CONTAINER(tableBases);
1042 }
1043
1044 void
1045 Gicv3Its::unserialize(CheckpointIn & cp)
1046 {
1047     UNSERIALIZE_SCALAR(gitsControl);
1048     UNSERIALIZE_SCALAR(gitsTyper);
1049     UNSERIALIZE_SCALAR(gitsCbaser);
1050     UNSERIALIZE_SCALAR(gitsCreadr);
1051     UNSERIALIZE_SCALAR(gitsCwriter);
1052     UNSERIALIZE_SCALAR(gitsIidr);
1053
1054     UNSERIALIZE_CONTAINER(tableBases);
1055 }
1056
1057 void
1058 Gicv3Its::incrementReadPointer()
1059 {
1060     // Make the reader point to the next element
1061     gitsCreadr.offset = gitsCreadr.offset + 1;
1062
1063     // Check for wrapping
1064     auto queue_end = (4096 * (gitsCbaser.size + 1));
1065
1066     if (gitsCreadr.offset == queue_end) {
1067         gitsCreadr.offset = 0;
1068     }
1069 }
1070
1071 void
1072 Gicv3Its::checkCommandQueue()
1073 {
1074     if (!gitsControl.enabled || !gitsCbaser.valid)
1075         return;
1076
1077     if (gitsCwriter.offset != gitsCreadr.offset) {
1078         // writer and reader pointing to different command
1079         // entries: queue not empty.
1080         DPRINTF(ITS, "Reading command from queue\n");
1081
1082         if (!pendingCommands) {
1083             auto *cmd_proc = new ItsCommand(*this);
1084
1085             runProcess(cmd_proc, nullptr);
1086         } else {
1087             DPRINTF(ITS, "Waiting for pending command to finish\n");
1088         }
1089     }
1090 }
1091
1092 Port &
1093 Gicv3Its::getPort(const std::string &if_name, PortID idx)
1094 {
1095     if (if_name == "dma") {
1096         return dmaPort;
1097     }
1098     return BasicPioDevice::getPort(if_name, idx);
1099 }
1100
1101 void
1102 Gicv3Its::recvReqRetry()
1103 {
1104     assert(!packetsToRetry.empty());
1105
1106     while (!packetsToRetry.empty()) {
1107         ItsAction a = packetsToRetry.front();
1108
1109         assert(a.type == ItsActionType::SEND_REQ);
1110
1111         if (!dmaPort.sendTimingReq(a.pkt))
1112             break;
1113
1114         packetsToRetry.pop();
1115     }
1116 }
1117
1118 bool
1119 Gicv3Its::recvTimingResp(PacketPtr pkt)
1120 {
1121     // @todo: We need to pay for this and not just zero it out
1122     pkt->headerDelay = pkt->payloadDelay = 0;
1123
1124     ItsProcess *proc =
1125         safe_cast<ItsProcess *>(pkt->popSenderState());
1126
1127     runProcessTiming(proc, pkt);
1128
1129     return true;
1130 }
1131
1132 ItsAction
1133 Gicv3Its::runProcess(ItsProcess *proc, PacketPtr pkt)
1134 {
1135     if (sys->isAtomicMode()) {
1136         return runProcessAtomic(proc, pkt);
1137     } else if (sys->isTimingMode()) {
1138         return runProcessTiming(proc, pkt);
1139     } else {
1140         panic("Not in timing or atomic mode\n");
1141     }
1142 }
1143
1144 ItsAction
1145 Gicv3Its::runProcessTiming(ItsProcess *proc, PacketPtr pkt)
1146 {
1147     ItsAction action = proc->run(pkt);
1148
1149     switch (action.type) {
1150       case ItsActionType::SEND_REQ:
1151         action.pkt->pushSenderState(proc);
1152
1153         if (packetsToRetry.empty() &&
1154             dmaPort.sendTimingReq(action.pkt)) {
1155
1156         } else {
1157             packetsToRetry.push(action);
1158         }
1159         break;
1160
1161       case ItsActionType::TERMINATE:
1162         delete proc;
1163         if (!pendingCommands && !commandEvent.scheduled()) {
1164             schedule(commandEvent, clockEdge());
1165         }
1166         break;
1167
1168       default:
1169         panic("Unknown action\n");
1170     }
1171
1172     return action;
1173 }
1174
1175 ItsAction
1176 Gicv3Its::runProcessAtomic(ItsProcess *proc, PacketPtr pkt)
1177 {
1178     ItsAction action;
1179     Tick delay = 0;
1180     bool terminate = false;
1181
1182     do {
1183         action = proc->run(pkt);
1184
1185         switch (action.type) {
1186           case ItsActionType::SEND_REQ:
1187             delay += dmaPort.sendAtomic(action.pkt);
1188             pkt = action.pkt;
1189             break;
1190
1191           case ItsActionType::TERMINATE:
1192             delete proc;
1193             terminate = true;
1194             break;
1195
1196           default:
1197             panic("Unknown action\n");
1198         }
1199
1200     } while (!terminate);
1201
1202     action.delay = delay;
1203
1204     return action;
1205 }
1206
1207 void
1208 Gicv3Its::translate(PacketPtr pkt)
1209 {
1210     DPRINTF(ITS, "Starting Translation Request\n");
1211
1212     auto *proc = new ItsTranslation(*this);
1213     runProcess(proc, pkt);
1214 }
1215
1216 Gicv3Redistributor*
1217 Gicv3Its::getRedistributor(uint64_t rd_base)
1218 {
1219     if (gitsTyper.pta == 1) {
1220         // RDBase is a redistributor address
1221         return gic->getRedistributorByAddr(rd_base << 16);
1222     } else {
1223         // RDBase is a redistributor number
1224         return gic->getRedistributor(rd_base);
1225     }
1226 }
1227
1228 Addr
1229 Gicv3Its::pageAddress(Gicv3Its::ItsTables table)
1230 {
1231     auto base_it = std::find_if(
1232         tableBases.begin(), tableBases.end(),
1233         [table] (const BASER &b) { return b.type == table; }
1234     );
1235
1236     panic_if(base_it == tableBases.end(),
1237         "ITS Table not recognised\n");
1238
1239     const BASER base = *base_it;
1240
1241     // real address depends on page size
1242     switch (base.pageSize) {
1243       case SIZE_4K:
1244       case SIZE_16K:
1245         return mbits(base, 47, 12);
1246       case SIZE_64K:
1247         return mbits(base, 47, 16) | (bits(base, 15, 12) << 48);
1248       default:
1249         panic("Unsupported page size\n");
1250     }
1251 }
1252
1253 void
1254 Gicv3Its::moveAllPendingState(
1255     Gicv3Redistributor *rd1, Gicv3Redistributor *rd2)
1256 {
1257     const uint64_t largest_lpi_id = 1ULL << (rd1->lpiIDBits + 1);
1258     uint8_t lpi_pending_table[largest_lpi_id / 8];
1259
1260     // Copying the pending table from redistributor 1 to redistributor 2
1261     rd1->memProxy->readBlob(
1262         rd1->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,
1263         sizeof(lpi_pending_table));
1264
1265     rd2->memProxy->writeBlob(
1266         rd2->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,
1267         sizeof(lpi_pending_table));
1268
1269     // Clearing pending table in redistributor 2
1270     rd1->memProxy->memsetBlob(
1271         rd1->lpiPendingTablePtr,
1272         0, sizeof(lpi_pending_table));
1273
1274     rd2->updateAndInformCPUInterface();
1275 }
1276
1277 Gicv3Its *
1278 Gicv3ItsParams::create()
1279 {
1280     return new Gicv3Its(this);
1281 }