src/dev/arm/gic_v3_its.cc

   1 /*
   2  * Copyright (c) 2019 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include "dev/arm/gic_v3_its.hh"
  39
  40 #include "debug/AddrRanges.hh"
  41 #include "debug/Drain.hh"
  42 #include "debug/GIC.hh"
  43 #include "debug/ITS.hh"
  44 #include "dev/arm/gic_v3.hh"
  45 #include "dev/arm/gic_v3_distributor.hh"
  46 #include "dev/arm/gic_v3_redistributor.hh"
  47 #include "mem/packet_access.hh"
  48
  49 #define COMMAND(x, method) { x, DispatchEntry(#x, method) }
  50
  51 const AddrRange Gicv3Its::GITS_BASER(0x0100, 0x0140);
  52
  53 const uint32_t Gicv3Its::CTLR_QUIESCENT = 0x80000000;
  54
  55 ItsProcess::ItsProcess(Gicv3Its &_its)
  56   : its(_its), coroutine(nullptr)
  57 {
  58 }
  59
  60 ItsProcess::~ItsProcess()
  61 {
  62 }
  63
  64 void
  65 ItsProcess::reinit()
  66 {
  67     coroutine.reset(new Coroutine(
  68         std::bind(&ItsProcess::main, this, std::placeholders::_1)));
  69 }
  70
  71 const std::string
  72 ItsProcess::name() const
  73 {
  74     return its.name();
  75 }
  76
  77 ItsAction
  78 ItsProcess::run(PacketPtr pkt)
  79 {
  80     assert(coroutine != nullptr);
  81     assert(*coroutine);
  82     return (*coroutine)(pkt).get();
  83 }
  84
  85 void
  86 ItsProcess::doRead(Yield &yield, Addr addr, void *ptr, size_t size)
  87 {
  88     ItsAction a;
  89     a.type = ItsActionType::SEND_REQ;
  90
  91     RequestPtr req = std::make_shared<Request>(
  92         addr, size, 0, its.requestorId);
  93
  94     req->taskId(ContextSwitchTaskId::DMA);
  95
  96     a.pkt = new Packet(req, MemCmd::ReadReq);
  97     a.pkt->dataStatic(ptr);
  98
  99     a.delay = 0;
 100
 101     PacketPtr pkt = yield(a).get();
 102
 103     assert(pkt);
 104     assert(pkt->getSize() >= size);
 105
 106     delete pkt;
 107 }
 108
 109 void
 110 ItsProcess::doWrite(Yield &yield, Addr addr, void *ptr, size_t size)
 111 {
 112     ItsAction a;
 113     a.type = ItsActionType::SEND_REQ;
 114
 115     RequestPtr req = std::make_shared<Request>(
 116         addr, size, 0, its.requestorId);
 117
 118     req->taskId(ContextSwitchTaskId::DMA);
 119
 120     a.pkt = new Packet(req, MemCmd::WriteReq);
 121     a.pkt->dataStatic(ptr);
 122
 123     a.delay = 0;
 124
 125     PacketPtr pkt = yield(a).get();
 126
 127     assert(pkt);
 128     assert(pkt->getSize() >= size);
 129
 130     delete pkt;
 131 }
 132
 133 void
 134 ItsProcess::terminate(Yield &yield)
 135 {
 136     ItsAction a;
 137     a.type = ItsActionType::TERMINATE;
 138     a.pkt = NULL;
 139     a.delay = 0;
 140     yield(a);
 141 }
 142
 143 void
 144 ItsProcess::writeDeviceTable(Yield &yield, uint32_t device_id, DTE dte)
 145 {
 146     const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);
 147     const Addr address = base + (device_id * sizeof(dte));
 148
 149     DPRINTF(ITS, "Writing DTE at address %#x: %#x\n", address, dte);
 150
 151     doWrite(yield, address, &dte, sizeof(dte));
 152 }
 153
 154 void
 155 ItsProcess::writeIrqTranslationTable(
 156     Yield &yield, const Addr itt_base, uint32_t event_id, ITTE itte)
 157 {
 158     const Addr address = itt_base + (event_id * sizeof(itte));
 159
 160     doWrite(yield, address, &itte, sizeof(itte));
 161
 162     DPRINTF(ITS, "Writing ITTE at address %#x: %#x\n", address, itte);
 163 }
 164
 165 void
 166 ItsProcess::writeIrqCollectionTable(
 167     Yield &yield, uint32_t collection_id, CTE cte)
 168 {
 169     const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);
 170     const Addr address = base + (collection_id * sizeof(cte));
 171
 172     doWrite(yield, address, &cte, sizeof(cte));
 173
 174     DPRINTF(ITS, "Writing CTE at address %#x: %#x\n", address, cte);
 175 }
 176
 177 uint64_t
 178 ItsProcess::readDeviceTable(Yield &yield, uint32_t device_id)
 179 {
 180     uint64_t dte;
 181     const Addr base = its.pageAddress(Gicv3Its::DEVICE_TABLE);
 182     const Addr address = base + (device_id * sizeof(dte));
 183
 184     doRead(yield, address, &dte, sizeof(dte));
 185
 186     DPRINTF(ITS, "Reading DTE at address %#x: %#x\n", address, dte);
 187     return dte;
 188 }
 189
 190 uint64_t
 191 ItsProcess::readIrqTranslationTable(
 192     Yield &yield, const Addr itt_base, uint32_t event_id)
 193 {
 194     uint64_t itte;
 195     const Addr address = itt_base + (event_id * sizeof(itte));
 196
 197     doRead(yield, address, &itte, sizeof(itte));
 198
 199     DPRINTF(ITS, "Reading ITTE at address %#x: %#x\n", address, itte);
 200     return itte;
 201 }
 202
 203 uint64_t
 204 ItsProcess::readIrqCollectionTable(Yield &yield, uint32_t collection_id)
 205 {
 206     uint64_t cte;
 207     const Addr base = its.pageAddress(Gicv3Its::COLLECTION_TABLE);
 208     const Addr address = base + (collection_id * sizeof(cte));
 209
 210     doRead(yield, address, &cte, sizeof(cte));
 211
 212     DPRINTF(ITS, "Reading CTE at address %#x: %#x\n", address, cte);
 213     return cte;
 214 }
 215
 216 ItsTranslation::ItsTranslation(Gicv3Its &_its)
 217   : ItsProcess(_its)
 218 {
 219     reinit();
 220     its.pendingTranslations++;
 221     its.gitsControl.quiescent = 0;
 222 }
 223
 224 ItsTranslation::~ItsTranslation()
 225 {
 226     assert(its.pendingTranslations >= 1);
 227     its.pendingTranslations--;
 228     if (!its.pendingTranslations && !its.pendingCommands)
 229         its.gitsControl.quiescent = 1;
 230 }
 231
 232 void
 233 ItsTranslation::main(Yield &yield)
 234 {
 235     PacketPtr pkt = yield.get();
 236
 237     const uint32_t device_id = pkt->req->streamId();
 238     const uint32_t event_id = pkt->getLE<uint32_t>();
 239
 240     auto result = translateLPI(yield, device_id, event_id);
 241
 242     uint32_t intid = result.first;
 243     Gicv3Redistributor *redist = result.second;
 244
 245     // Set the LPI in the redistributor
 246     redist->setClrLPI(intid, true);
 247
 248     // Update the value in GITS_TRANSLATER only once we know
 249     // there was no error in the tranlation process (before
 250     // terminating the translation
 251     its.gitsTranslater = event_id;
 252
 253     terminate(yield);
 254 }
 255
 256 std::pair<uint32_t, Gicv3Redistributor *>
 257 ItsTranslation::translateLPI(Yield &yield, uint32_t device_id,
 258                              uint32_t event_id)
 259 {
 260     if (its.deviceOutOfRange(device_id)) {
 261         terminate(yield);
 262     }
 263
 264     DTE dte = readDeviceTable(yield, device_id);
 265
 266     if (!dte.valid || its.idOutOfRange(event_id, dte.ittRange)) {
 267         terminate(yield);
 268     }
 269
 270     ITTE itte = readIrqTranslationTable(yield, dte.ittAddress, event_id);
 271     const auto collection_id = itte.icid;
 272
 273     if (!itte.valid || its.collectionOutOfRange(collection_id)) {
 274         terminate(yield);
 275     }
 276
 277     CTE cte = readIrqCollectionTable(yield, collection_id);
 278
 279     if (!cte.valid) {
 280         terminate(yield);
 281     }
 282
 283     // Returning the INTID and the target Redistributor
 284     return std::make_pair(itte.intNum, its.getRedistributor(cte));
 285 }
 286
 287 ItsCommand::DispatchTable ItsCommand::cmdDispatcher =
 288 {
 289     COMMAND(CLEAR, &ItsCommand::clear),
 290     COMMAND(DISCARD, &ItsCommand::discard),
 291     COMMAND(INT, &ItsCommand::doInt),
 292     COMMAND(INV, &ItsCommand::inv),
 293     COMMAND(INVALL, &ItsCommand::invall),
 294     COMMAND(MAPC, &ItsCommand::mapc),
 295     COMMAND(MAPD, &ItsCommand::mapd),
 296     COMMAND(MAPI, &ItsCommand::mapi),
 297     COMMAND(MAPTI, &ItsCommand::mapti),
 298     COMMAND(MOVALL, &ItsCommand::movall),
 299     COMMAND(MOVI, &ItsCommand::movi),
 300     COMMAND(SYNC, &ItsCommand::sync),
 301     COMMAND(VINVALL, &ItsCommand::vinvall),
 302     COMMAND(VMAPI, &ItsCommand::vmapi),
 303     COMMAND(VMAPP, &ItsCommand::vmapp),
 304     COMMAND(VMAPTI, &ItsCommand::vmapti),
 305     COMMAND(VMOVI, &ItsCommand::vmovi),
 306     COMMAND(VMOVP, &ItsCommand::vmovp),
 307     COMMAND(VSYNC, &ItsCommand::vsync),
 308 };
 309
 310 ItsCommand::ItsCommand(Gicv3Its &_its)
 311   : ItsProcess(_its)
 312 {
 313     reinit();
 314     its.pendingCommands = true;
 315
 316     its.gitsControl.quiescent = 0;
 317 }
 318
 319 ItsCommand::~ItsCommand()
 320 {
 321     its.pendingCommands = false;
 322
 323     if (!its.pendingTranslations)
 324         its.gitsControl.quiescent = 1;
 325 }
 326
 327 std::string
 328 ItsCommand::commandName(uint32_t cmd)
 329 {
 330     const auto entry = cmdDispatcher.find(cmd);
 331     return entry != cmdDispatcher.end() ? entry->second.name : "INVALID";
 332 }
 333
 334 void
 335 ItsCommand::main(Yield &yield)
 336 {
 337     ItsAction a;
 338     a.type = ItsActionType::INITIAL_NOP;
 339     a.pkt = nullptr;
 340     a.delay = 0;
 341     yield(a);
 342
 343     while (its.gitsCwriter.offset != its.gitsCreadr.offset) {
 344         CommandEntry command;
 345
 346         // Reading the command from CMDQ
 347         readCommand(yield, command);
 348
 349         processCommand(yield, command);
 350
 351         its.incrementReadPointer();
 352     }
 353
 354     terminate(yield);
 355 }
 356
 357 void
 358 ItsCommand::readCommand(Yield &yield, CommandEntry &command)
 359 {
 360     // read the command pointed by GITS_CREADR
 361     const Addr cmd_addr =
 362         (its.gitsCbaser.physAddr << 12) + (its.gitsCreadr.offset << 5);
 363
 364     doRead(yield, cmd_addr, &command, sizeof(command));
 365
 366     DPRINTF(ITS, "Command %s read from queue at address: %#x\n",
 367             commandName(command.type), cmd_addr);
 368     DPRINTF(ITS, "dw0: %#x dw1: %#x dw2: %#x dw3: %#x\n",
 369             command.raw[0], command.raw[1], command.raw[2], command.raw[3]);
 370 }
 371
 372 void
 373 ItsCommand::processCommand(Yield &yield, CommandEntry &command)
 374 {
 375     const auto entry = cmdDispatcher.find(command.type);
 376
 377     if (entry != cmdDispatcher.end()) {
 378         // Execute the command
 379         entry->second.exec(this, yield, command);
 380     } else {
 381         panic("Unrecognized command type: %u", command.type);
 382     }
 383 }
 384
 385 void
 386 ItsCommand::clear(Yield &yield, CommandEntry &command)
 387 {
 388     if (deviceOutOfRange(command)) {
 389         its.incrementReadPointer();
 390         terminate(yield);
 391     }
 392
 393     DTE dte = readDeviceTable(yield, command.deviceId);
 394
 395     if (!dte.valid || idOutOfRange(command, dte)) {
 396         its.incrementReadPointer();
 397         terminate(yield);
 398     }
 399
 400     ITTE itte = readIrqTranslationTable(
 401         yield, dte.ittAddress, command.eventId);
 402
 403     if (!itte.valid) {
 404         its.incrementReadPointer();
 405         terminate(yield);
 406     }
 407
 408     const auto collection_id = itte.icid;
 409     CTE cte = readIrqCollectionTable(yield, collection_id);
 410
 411     if (!cte.valid) {
 412         its.incrementReadPointer();
 413         terminate(yield);
 414     }
 415
 416     // Clear the LPI in the redistributor
 417     its.getRedistributor(cte)->setClrLPI(itte.intNum, false);
 418 }
 419
 420 void
 421 ItsCommand::discard(Yield &yield, CommandEntry &command)
 422 {
 423     if (deviceOutOfRange(command)) {
 424         its.incrementReadPointer();
 425         terminate(yield);
 426     }
 427
 428     DTE dte = readDeviceTable(yield, command.deviceId);
 429
 430     if (!dte.valid || idOutOfRange(command, dte)) {
 431         its.incrementReadPointer();
 432         terminate(yield);
 433     }
 434
 435     ITTE itte = readIrqTranslationTable(
 436         yield, dte.ittAddress, command.eventId);
 437
 438     if (!itte.valid) {
 439         its.incrementReadPointer();
 440         terminate(yield);
 441     }
 442
 443     const auto collection_id = itte.icid;
 444     Gicv3Its::CTE cte = readIrqCollectionTable(yield, collection_id);
 445
 446     if (!cte.valid) {
 447         its.incrementReadPointer();
 448         terminate(yield);
 449     }
 450
 451     its.getRedistributor(cte)->setClrLPI(itte.intNum, false);
 452
 453     // Then removes the mapping from the ITT (invalidating)
 454     itte.valid = 0;
 455     writeIrqTranslationTable(
 456         yield, dte.ittAddress, command.eventId, itte);
 457 }
 458
 459 void
 460 ItsCommand::doInt(Yield &yield, CommandEntry &command)
 461 {
 462     if (deviceOutOfRange(command)) {
 463         its.incrementReadPointer();
 464         terminate(yield);
 465     }
 466
 467     DTE dte = readDeviceTable(yield, command.deviceId);
 468
 469     if (!dte.valid || idOutOfRange(command, dte)) {
 470         its.incrementReadPointer();
 471         terminate(yield);
 472     }
 473
 474     ITTE itte = readIrqTranslationTable(
 475         yield, dte.ittAddress, command.eventId);
 476
 477     if (!itte.valid) {
 478         its.incrementReadPointer();
 479         terminate(yield);
 480     }
 481
 482     const auto collection_id = itte.icid;
 483     CTE cte = readIrqCollectionTable(yield, collection_id);
 484
 485     if (!cte.valid) {
 486         its.incrementReadPointer();
 487         terminate(yield);
 488     }
 489
 490     // Set the LPI in the redistributor
 491     its.getRedistributor(cte)->setClrLPI(itte.intNum, true);
 492 }
 493
 494 void
 495 ItsCommand::inv(Yield &yield, CommandEntry &command)
 496 {
 497     if (deviceOutOfRange(command)) {
 498         its.incrementReadPointer();
 499         terminate(yield);
 500     }
 501
 502     DTE dte = readDeviceTable(yield, command.deviceId);
 503
 504     if (!dte.valid || idOutOfRange(command, dte)) {
 505         its.incrementReadPointer();
 506         terminate(yield);
 507     }
 508
 509     ITTE itte = readIrqTranslationTable(
 510         yield, dte.ittAddress, command.eventId);
 511
 512     if (!itte.valid) {
 513         its.incrementReadPointer();
 514         terminate(yield);
 515     }
 516
 517     const auto collection_id = itte.icid;
 518     CTE cte = readIrqCollectionTable(yield, collection_id);
 519
 520     if (!cte.valid) {
 521         its.incrementReadPointer();
 522         terminate(yield);
 523     }
 524     // Do nothing since caching is currently not supported in
 525     // Redistributor
 526 }
 527
 528 void
 529 ItsCommand::invall(Yield &yield, CommandEntry &command)
 530 {
 531     if (collectionOutOfRange(command)) {
 532         its.incrementReadPointer();
 533         terminate(yield);
 534     }
 535
 536     const auto icid = bits(command.raw[2], 15, 0);
 537
 538     CTE cte = readIrqCollectionTable(yield, icid);
 539
 540     if (!cte.valid) {
 541         its.incrementReadPointer();
 542         terminate(yield);
 543     }
 544     // Do nothing since caching is currently not supported in
 545     // Redistributor
 546 }
 547
 548 void
 549 ItsCommand::mapc(Yield &yield, CommandEntry &command)
 550 {
 551     if (collectionOutOfRange(command)) {
 552         its.incrementReadPointer();
 553         terminate(yield);
 554     }
 555
 556     CTE cte = 0;
 557     cte.valid = bits(command.raw[2], 63);
 558     cte.rdBase = bits(command.raw[2], 50, 16);
 559
 560     const auto icid = bits(command.raw[2], 15, 0);
 561
 562     writeIrqCollectionTable(yield, icid, cte);
 563 }
 564
 565 void
 566 ItsCommand::mapd(Yield &yield, CommandEntry &command)
 567 {
 568     if (deviceOutOfRange(command) || sizeOutOfRange(command)) {
 569         its.incrementReadPointer();
 570         terminate(yield);
 571     }
 572
 573     DTE dte = 0;
 574     dte.valid = bits(command.raw[2], 63);
 575     dte.ittAddress = mbits(command.raw[2], 51, 8);
 576     dte.ittRange = bits(command.raw[1], 4, 0);
 577
 578     writeDeviceTable(yield, command.deviceId, dte);
 579 }
 580
 581 void
 582 ItsCommand::mapi(Yield &yield, CommandEntry &command)
 583 {
 584     if (deviceOutOfRange(command)) {
 585         its.incrementReadPointer();
 586         terminate(yield);
 587     }
 588
 589     if (collectionOutOfRange(command)) {
 590         its.incrementReadPointer();
 591         terminate(yield);
 592     }
 593
 594     DTE dte = readDeviceTable(yield, command.deviceId);
 595
 596     if (!dte.valid || idOutOfRange(command, dte) ||
 597         its.lpiOutOfRange(command.eventId)) {
 598
 599         its.incrementReadPointer();
 600         terminate(yield);
 601     }
 602
 603     Gicv3Its::ITTE itte = readIrqTranslationTable(
 604         yield, dte.ittAddress, command.eventId);
 605
 606     itte.valid = 1;
 607     itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;
 608     itte.intNum = command.eventId;
 609     itte.icid = bits(command.raw[2], 15, 0);
 610
 611     writeIrqTranslationTable(
 612         yield, dte.ittAddress, command.eventId, itte);
 613 }
 614
 615 void
 616 ItsCommand::mapti(Yield &yield, CommandEntry &command)
 617 {
 618     if (deviceOutOfRange(command)) {
 619         its.incrementReadPointer();
 620         terminate(yield);
 621     }
 622
 623     if (collectionOutOfRange(command)) {
 624         its.incrementReadPointer();
 625         terminate(yield);
 626     }
 627
 628     DTE dte = readDeviceTable(yield, command.deviceId);
 629
 630     const auto pintid = bits(command.raw[1], 63, 32);
 631
 632     if (!dte.valid || idOutOfRange(command, dte) ||
 633         its.lpiOutOfRange(pintid)) {
 634
 635         its.incrementReadPointer();
 636         terminate(yield);
 637     }
 638
 639     ITTE itte = readIrqTranslationTable(
 640         yield, dte.ittAddress, command.eventId);
 641
 642     itte.valid = 1;
 643     itte.intType = Gicv3Its::PHYSICAL_INTERRUPT;
 644     itte.intNum = pintid;
 645     itte.icid = bits(command.raw[2], 15, 0);
 646
 647     writeIrqTranslationTable(
 648         yield, dte.ittAddress, command.eventId, itte);
 649 }
 650
 651 void
 652 ItsCommand::movall(Yield &yield, CommandEntry &command)
 653 {
 654     const uint64_t rd1 = bits(command.raw[2], 50, 16);
 655     const uint64_t rd2 = bits(command.raw[3], 50, 16);
 656
 657     if (rd1 != rd2) {
 658         Gicv3Redistributor * redist1 = its.getRedistributor(rd1);
 659         Gicv3Redistributor * redist2 = its.getRedistributor(rd2);
 660
 661         its.moveAllPendingState(redist1, redist2);
 662     }
 663 }
 664
 665 void
 666 ItsCommand::movi(Yield &yield, CommandEntry &command)
 667 {
 668     if (deviceOutOfRange(command)) {
 669         its.incrementReadPointer();
 670         terminate(yield);
 671     }
 672
 673     if (collectionOutOfRange(command)) {
 674         its.incrementReadPointer();
 675         terminate(yield);
 676     }
 677
 678     DTE dte = readDeviceTable(yield, command.deviceId);
 679
 680     if (!dte.valid || idOutOfRange(command, dte)) {
 681         its.incrementReadPointer();
 682         terminate(yield);
 683     }
 684
 685     ITTE itte = readIrqTranslationTable(
 686         yield, dte.ittAddress, command.eventId);
 687
 688     if (!itte.valid || itte.intType == Gicv3Its::VIRTUAL_INTERRUPT) {
 689         its.incrementReadPointer();
 690         terminate(yield);
 691     }
 692
 693     const auto collection_id1 = itte.icid;
 694     CTE cte1 = readIrqCollectionTable(yield, collection_id1);
 695
 696     if (!cte1.valid) {
 697         its.incrementReadPointer();
 698         terminate(yield);
 699     }
 700
 701     const auto collection_id2 = bits(command.raw[2], 15, 0);
 702     CTE cte2 = readIrqCollectionTable(yield, collection_id2);
 703
 704     if (!cte2.valid) {
 705         its.incrementReadPointer();
 706         terminate(yield);
 707     }
 708
 709     Gicv3Redistributor *first_redist = its.getRedistributor(cte1);
 710     Gicv3Redistributor *second_redist = its.getRedistributor(cte2);
 711
 712     if (second_redist != first_redist) {
 713         // move pending state of the interrupt from one redistributor
 714         // to the other.
 715         if (first_redist->isPendingLPI(itte.intNum)) {
 716             first_redist->setClrLPI(itte.intNum, false);
 717             second_redist->setClrLPI(itte.intNum, true);
 718         }
 719     }
 720
 721     itte.icid = collection_id2;
 722     writeIrqTranslationTable(
 723         yield, dte.ittAddress, command.eventId, itte);
 724 }
 725
 726 void
 727 ItsCommand::sync(Yield &yield, CommandEntry &command)
 728 {
 729     warn("ITS %s command unimplemented", __func__);
 730 }
 731
 732 void
 733 ItsCommand::vinvall(Yield &yield, CommandEntry &command)
 734 {
 735     panic("ITS %s command unimplemented", __func__);
 736 }
 737
 738 void
 739 ItsCommand::vmapi(Yield &yield, CommandEntry &command)
 740 {
 741     panic("ITS %s command unimplemented", __func__);
 742 }
 743
 744 void
 745 ItsCommand::vmapp(Yield &yield, CommandEntry &command)
 746 {
 747     panic("ITS %s command unimplemented", __func__);
 748 }
 749
 750 void
 751 ItsCommand::vmapti(Yield &yield, CommandEntry &command)
 752 {
 753     panic("ITS %s command unimplemented", __func__);
 754 }
 755
 756 void
 757 ItsCommand::vmovi(Yield &yield, CommandEntry &command)
 758 {
 759     panic("ITS %s command unimplemented", __func__);
 760 }
 761
 762 void
 763 ItsCommand::vmovp(Yield &yield, CommandEntry &command)
 764 {
 765     panic("ITS %s command unimplemented", __func__);
 766 }
 767
 768 void
 769 ItsCommand::vsync(Yield &yield, CommandEntry &command)
 770 {
 771     panic("ITS %s command unimplemented", __func__);
 772 }
 773
 774 Gicv3Its::Gicv3Its(const Gicv3ItsParams &params)
 775  : BasicPioDevice(params, params.pio_size),
 776    dmaPort(name() + ".dma", *this),
 777    gitsControl(CTLR_QUIESCENT),
 778    gitsTyper(params.gits_typer),
 779    gitsCbaser(0), gitsCreadr(0),
 780    gitsCwriter(0), gitsIidr(0),
 781    tableBases(NUM_BASER_REGS, 0),
 782    requestorId(params.system->getRequestorId(this)),
 783    gic(nullptr),
 784    commandEvent([this] { checkCommandQueue(); }, name()),
 785    pendingCommands(false),
 786    pendingTranslations(0)
 787 {
 788     BASER device_baser = 0;
 789     device_baser.type = DEVICE_TABLE;
 790     device_baser.entrySize = sizeof(uint64_t) - 1;
 791     tableBases[0] = device_baser;
 792
 793     BASER icollect_baser = 0;
 794     icollect_baser.type = COLLECTION_TABLE;
 795     icollect_baser.entrySize = sizeof(uint64_t) - 1;
 796     tableBases[1] = icollect_baser;
 797 }
 798
 799 void
 800 Gicv3Its::setGIC(Gicv3 *_gic)
 801 {
 802     assert(!gic);
 803     gic = _gic;
 804 }
 805
 806 AddrRangeList
 807 Gicv3Its::getAddrRanges() const
 808 {
 809     assert(pioSize != 0);
 810     AddrRangeList ranges;
 811     DPRINTF(AddrRanges, "registering range: %#x-%#x\n", pioAddr, pioSize);
 812     ranges.push_back(RangeSize(pioAddr, pioSize));
 813     return ranges;
 814 }
 815
 816 Tick
 817 Gicv3Its::read(PacketPtr pkt)
 818 {
 819     const Addr addr = pkt->getAddr() - pioAddr;
 820     uint64_t value = 0;
 821
 822     DPRINTF(GIC, "%s register at addr: %#x\n", __func__, addr);
 823
 824     switch (addr) {
 825       case GITS_CTLR:
 826         value = gitsControl;
 827         break;
 828
 829       case GITS_IIDR:
 830         value = gitsIidr;
 831         break;
 832
 833       case GITS_TYPER:
 834         value = gitsTyper;
 835         break;
 836
 837       case GITS_TYPER + 4:
 838         value = gitsTyper.high;
 839         break;
 840
 841       case GITS_CBASER:
 842         value = gitsCbaser;
 843         break;
 844
 845       case GITS_CBASER + 4:
 846         value = gitsCbaser.high;
 847         break;
 848
 849       case GITS_CWRITER:
 850         value = gitsCwriter;
 851         break;
 852
 853       case GITS_CWRITER + 4:
 854         value = gitsCwriter.high;
 855         break;
 856
 857       case GITS_CREADR:
 858         value = gitsCreadr;
 859         break;
 860
 861       case GITS_CREADR + 4:
 862         value = gitsCreadr.high;
 863         break;
 864
 865       case GITS_PIDR2:
 866         value = gic->getDistributor()->gicdPidr2;
 867         break;
 868
 869       case GITS_TRANSLATER:
 870         value = gitsTranslater;
 871         break;
 872
 873       default:
 874         if (GITS_BASER.contains(addr)) {
 875             auto relative_addr = addr - GITS_BASER.start();
 876             auto baser_index = relative_addr / sizeof(uint64_t);
 877
 878             value = tableBases[baser_index];
 879             break;
 880         } else {
 881             panic("Unrecognized register access\n");
 882         }
 883     }
 884
 885     pkt->setUintX(value, ByteOrder::little);
 886     pkt->makeAtomicResponse();
 887     return pioDelay;
 888 }
 889
 890 Tick
 891 Gicv3Its::write(PacketPtr pkt)
 892 {
 893     Addr addr = pkt->getAddr() - pioAddr;
 894
 895     DPRINTF(GIC, "%s register at addr: %#x\n", __func__, addr);
 896
 897     switch (addr) {
 898       case GITS_CTLR:
 899         assert(pkt->getSize() == sizeof(uint32_t));
 900         gitsControl = (pkt->getLE<uint32_t>() & ~CTLR_QUIESCENT);
 901         // We should check here if the ITS has been disabled, and if
 902         // that's the case, flush GICv3 caches to external memory.
 903         // This is not happening now, since LPI caching is not
 904         // currently implemented in gem5.
 905         break;
 906
 907       case GITS_IIDR:
 908         panic("GITS_IIDR is Read Only\n");
 909
 910       case GITS_TYPER:
 911         panic("GITS_TYPER is Read Only\n");
 912
 913       case GITS_CBASER:
 914         if (pkt->getSize() == sizeof(uint32_t)) {
 915             gitsCbaser.low = pkt->getLE<uint32_t>();
 916         } else {
 917             assert(pkt->getSize() == sizeof(uint64_t));
 918             gitsCbaser = pkt->getLE<uint64_t>();
 919         }
 920
 921         gitsCreadr = 0; // Cleared when CBASER gets written
 922
 923         checkCommandQueue();
 924         break;
 925
 926       case GITS_CBASER + 4:
 927         assert(pkt->getSize() == sizeof(uint32_t));
 928         gitsCbaser.high = pkt->getLE<uint32_t>();
 929
 930         gitsCreadr = 0; // Cleared when CBASER gets written
 931
 932         checkCommandQueue();
 933         break;
 934
 935       case GITS_CWRITER:
 936         if (pkt->getSize() == sizeof(uint32_t)) {
 937             gitsCwriter.low = pkt->getLE<uint32_t>();
 938         } else {
 939             assert(pkt->getSize() == sizeof(uint64_t));
 940             gitsCwriter = pkt->getLE<uint64_t>();
 941         }
 942
 943         checkCommandQueue();
 944         break;
 945
 946       case GITS_CWRITER + 4:
 947         assert(pkt->getSize() == sizeof(uint32_t));
 948         gitsCwriter.high = pkt->getLE<uint32_t>();
 949
 950         checkCommandQueue();
 951         break;
 952
 953       case GITS_CREADR:
 954         panic("GITS_READR is Read Only\n");
 955
 956       case GITS_TRANSLATER:
 957         if (gitsControl.enabled) {
 958             translate(pkt);
 959         }
 960         break;
 961
 962       default:
 963         if (GITS_BASER.contains(addr)) {
 964             auto relative_addr = addr - GITS_BASER.start();
 965             auto baser_index = relative_addr / sizeof(uint64_t);
 966
 967             const uint64_t table_base = tableBases[baser_index];
 968             const uint64_t w_mask = tableBases[baser_index].type ?
 969                 BASER_WMASK : BASER_WMASK_UNIMPL;
 970             const uint64_t val = pkt->getLE<uint64_t>() & w_mask;
 971
 972             tableBases[baser_index] = table_base | val;
 973             break;
 974         } else {
 975             panic("Unrecognized register access\n");
 976         }
 977     }
 978
 979     pkt->makeAtomicResponse();
 980     return pioDelay;
 981 }
 982
 983 bool
 984 Gicv3Its::idOutOfRange(uint32_t event_id, uint8_t itt_range) const
 985 {
 986     const uint32_t id_bits = gitsTyper.idBits;
 987     return event_id >= (1ULL << (id_bits + 1)) ||
 988         event_id >= ((1ULL << itt_range) + 1);
 989 }
 990
 991 bool
 992 Gicv3Its::deviceOutOfRange(uint32_t device_id) const
 993 {
 994     return device_id >= (1ULL << (gitsTyper.devBits + 1));
 995 }
 996
 997 bool
 998 Gicv3Its::sizeOutOfRange(uint32_t size) const
 999 {
1000     return size > gitsTyper.idBits;
1001 }
1002
1003 bool
1004 Gicv3Its::collectionOutOfRange(uint32_t collection_id) const
1005 {
1006     // If GITS_TYPER.CIL == 0, ITS supports 16-bit CollectionID
1007     // Otherwise, #bits is specified by GITS_TYPER.CIDbits
1008     const auto cid_bits = gitsTyper.cil == 0 ?
1009         16 : gitsTyper.cidBits + 1;
1010
1011     return collection_id >= (1ULL << cid_bits);
1012 }
1013
1014 bool
1015 Gicv3Its::lpiOutOfRange(uint32_t intid) const
1016 {
1017     return intid >= (1ULL << (Gicv3Distributor::IDBITS + 1)) ||
1018            (intid < Gicv3Redistributor::SMALLEST_LPI_ID &&
1019             intid != Gicv3::INTID_SPURIOUS);
1020 }
1021
1022 DrainState
1023 Gicv3Its::drain()
1024 {
1025     if (!pendingCommands && !pendingTranslations) {
1026         return DrainState::Drained;
1027     } else {
1028         DPRINTF(Drain, "GICv3 ITS not drained\n");
1029         return DrainState::Draining;
1030     }
1031 }
1032
1033 void
1034 Gicv3Its::serialize(CheckpointOut & cp) const
1035 {
1036     SERIALIZE_SCALAR(gitsControl);
1037     SERIALIZE_SCALAR(gitsTyper);
1038     SERIALIZE_SCALAR(gitsCbaser);
1039     SERIALIZE_SCALAR(gitsCreadr);
1040     SERIALIZE_SCALAR(gitsCwriter);
1041     SERIALIZE_SCALAR(gitsIidr);
1042
1043     SERIALIZE_CONTAINER(tableBases);
1044 }
1045
1046 void
1047 Gicv3Its::unserialize(CheckpointIn & cp)
1048 {
1049     UNSERIALIZE_SCALAR(gitsControl);
1050     UNSERIALIZE_SCALAR(gitsTyper);
1051     UNSERIALIZE_SCALAR(gitsCbaser);
1052     UNSERIALIZE_SCALAR(gitsCreadr);
1053     UNSERIALIZE_SCALAR(gitsCwriter);
1054     UNSERIALIZE_SCALAR(gitsIidr);
1055
1056     UNSERIALIZE_CONTAINER(tableBases);
1057 }
1058
1059 void
1060 Gicv3Its::incrementReadPointer()
1061 {
1062     // Make the reader point to the next element
1063     gitsCreadr.offset = gitsCreadr.offset + 1;
1064
1065     // Check for wrapping
1066     if (gitsCreadr.offset == maxCommands()) {
1067         gitsCreadr.offset = 0;
1068     }
1069 }
1070
1071 uint64_t
1072 Gicv3Its::maxCommands() const
1073 {
1074     return (4096 * (gitsCbaser.size + 1)) / sizeof(ItsCommand::CommandEntry);
1075 }
1076
1077 void
1078 Gicv3Its::checkCommandQueue()
1079 {
1080     if (!gitsControl.enabled || !gitsCbaser.valid)
1081         return;
1082
1083     // If GITS_CWRITER gets set by sw to a value bigger than the
1084     // allowed one, the command queue should stop processing commands
1085     // until the register gets reset to an allowed one
1086     if (gitsCwriter.offset >= maxCommands()) {
1087         return;
1088     }
1089
1090     if (gitsCwriter.offset != gitsCreadr.offset) {
1091         // writer and reader pointing to different command
1092         // entries: queue not empty.
1093         DPRINTF(ITS, "Reading command from queue\n");
1094
1095         if (!pendingCommands) {
1096             auto *cmd_proc = new ItsCommand(*this);
1097
1098             runProcess(cmd_proc, nullptr);
1099         } else {
1100             DPRINTF(ITS, "Waiting for pending command to finish\n");
1101         }
1102     }
1103 }
1104
1105 Port &
1106 Gicv3Its::getPort(const std::string &if_name, PortID idx)
1107 {
1108     if (if_name == "dma") {
1109         return dmaPort;
1110     }
1111     return BasicPioDevice::getPort(if_name, idx);
1112 }
1113
1114 void
1115 Gicv3Its::recvReqRetry()
1116 {
1117     assert(!packetsToRetry.empty());
1118
1119     while (!packetsToRetry.empty()) {
1120         ItsAction a = packetsToRetry.front();
1121
1122         assert(a.type == ItsActionType::SEND_REQ);
1123
1124         if (!dmaPort.sendTimingReq(a.pkt))
1125             break;
1126
1127         packetsToRetry.pop();
1128     }
1129 }
1130
1131 bool
1132 Gicv3Its::recvTimingResp(PacketPtr pkt)
1133 {
1134     // @todo: We need to pay for this and not just zero it out
1135     pkt->headerDelay = pkt->payloadDelay = 0;
1136
1137     ItsProcess *proc =
1138         safe_cast<ItsProcess *>(pkt->popSenderState());
1139
1140     runProcessTiming(proc, pkt);
1141
1142     return true;
1143 }
1144
1145 ItsAction
1146 Gicv3Its::runProcess(ItsProcess *proc, PacketPtr pkt)
1147 {
1148     if (sys->isAtomicMode()) {
1149         return runProcessAtomic(proc, pkt);
1150     } else if (sys->isTimingMode()) {
1151         return runProcessTiming(proc, pkt);
1152     } else {
1153         panic("Not in timing or atomic mode\n");
1154     }
1155 }
1156
1157 ItsAction
1158 Gicv3Its::runProcessTiming(ItsProcess *proc, PacketPtr pkt)
1159 {
1160     ItsAction action = proc->run(pkt);
1161
1162     switch (action.type) {
1163       case ItsActionType::SEND_REQ:
1164         action.pkt->pushSenderState(proc);
1165
1166         if (packetsToRetry.empty() &&
1167             dmaPort.sendTimingReq(action.pkt)) {
1168
1169         } else {
1170             packetsToRetry.push(action);
1171         }
1172         break;
1173
1174       case ItsActionType::TERMINATE:
1175         delete proc;
1176         if (!pendingCommands && !commandEvent.scheduled()) {
1177             schedule(commandEvent, clockEdge());
1178         }
1179         break;
1180
1181       default:
1182         panic("Unknown action\n");
1183     }
1184
1185     return action;
1186 }
1187
1188 ItsAction
1189 Gicv3Its::runProcessAtomic(ItsProcess *proc, PacketPtr pkt)
1190 {
1191     ItsAction action;
1192     Tick delay = 0;
1193     bool terminate = false;
1194
1195     do {
1196         action = proc->run(pkt);
1197
1198         switch (action.type) {
1199           case ItsActionType::SEND_REQ:
1200             delay += dmaPort.sendAtomic(action.pkt);
1201             pkt = action.pkt;
1202             break;
1203
1204           case ItsActionType::TERMINATE:
1205             delete proc;
1206             terminate = true;
1207             break;
1208
1209           default:
1210             panic("Unknown action\n");
1211         }
1212
1213     } while (!terminate);
1214
1215     action.delay = delay;
1216
1217     return action;
1218 }
1219
1220 void
1221 Gicv3Its::translate(PacketPtr pkt)
1222 {
1223     DPRINTF(ITS, "Starting Translation Request\n");
1224
1225     auto *proc = new ItsTranslation(*this);
1226     runProcess(proc, pkt);
1227 }
1228
1229 Gicv3Redistributor*
1230 Gicv3Its::getRedistributor(uint64_t rd_base)
1231 {
1232     if (gitsTyper.pta == 1) {
1233         // RDBase is a redistributor address
1234         return gic->getRedistributorByAddr(rd_base << 16);
1235     } else {
1236         // RDBase is a redistributor number
1237         return gic->getRedistributor(rd_base);
1238     }
1239 }
1240
1241 Addr
1242 Gicv3Its::pageAddress(Gicv3Its::ItsTables table)
1243 {
1244     auto base_it = std::find_if(
1245         tableBases.begin(), tableBases.end(),
1246         [table] (const BASER &b) { return b.type == table; }
1247     );
1248
1249     panic_if(base_it == tableBases.end(),
1250         "ITS Table not recognised\n");
1251
1252     const BASER base = *base_it;
1253
1254     // real address depends on page size
1255     switch (base.pageSize) {
1256       case SIZE_4K:
1257       case SIZE_16K:
1258         return mbits(base, 47, 12);
1259       case SIZE_64K:
1260         return mbits(base, 47, 16) | (bits(base, 15, 12) << 48);
1261       default:
1262         panic("Unsupported page size\n");
1263     }
1264 }
1265
1266 void
1267 Gicv3Its::moveAllPendingState(
1268     Gicv3Redistributor *rd1, Gicv3Redistributor *rd2)
1269 {
1270     const uint64_t largest_lpi_id = 1ULL << (rd1->lpiIDBits + 1);
1271     uint8_t lpi_pending_table[largest_lpi_id / 8];
1272
1273     // Copying the pending table from redistributor 1 to redistributor 2
1274     rd1->memProxy->readBlob(
1275         rd1->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,
1276         sizeof(lpi_pending_table));
1277
1278     rd2->memProxy->writeBlob(
1279         rd2->lpiPendingTablePtr, (uint8_t *)lpi_pending_table,
1280         sizeof(lpi_pending_table));
1281
1282     // Clearing pending table in redistributor 2
1283     rd1->memProxy->memsetBlob(
1284         rd1->lpiPendingTablePtr,
1285         0, sizeof(lpi_pending_table));
1286
1287     rd2->updateDistributor();
1288 }
1289
1290 Gicv3Its *
1291 Gicv3ItsParams::create() const
1292 {
1293     return new Gicv3Its(*this);
1294 }