src/dev/arm/smmu_v3.cc

   1 /*
   2  * Copyright (c) 2013, 2018-2019 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  *
  37  * Authors: Stan Czerniawski
  38  */
  39
  40 #include "dev/arm/smmu_v3.hh"
  41
  42 #include <cstddef>
  43 #include <cstring>
  44
  45 #include "base/bitfield.hh"
  46 #include "base/cast.hh"
  47 #include "base/logging.hh"
  48 #include "base/trace.hh"
  49 #include "base/types.hh"
  50 #include "debug/Checkpoint.hh"
  51 #include "debug/SMMUv3.hh"
  52 #include "dev/arm/smmu_v3_transl.hh"
  53 #include "mem/packet_access.hh"
  54 #include "sim/system.hh"
  55
  56 SMMUv3::SMMUv3(SMMUv3Params *params) :
  57     MemObject(params),
  58     system(*params->system),
  59     masterId(params->system->getMasterId(this)),
  60     masterPort(name() + ".master", *this),
  61     masterTableWalkPort(name() + ".master_walker", *this),
  62     controlPort(name() + ".control", *this, params->reg_map),
  63     tlb(params->tlb_entries, params->tlb_assoc, params->tlb_policy),
  64     configCache(params->cfg_entries, params->cfg_assoc, params->cfg_policy),
  65     ipaCache(params->ipa_entries, params->ipa_assoc, params->ipa_policy),
  66     walkCache({ { params->walk_S1L0, params->walk_S1L1,
  67                   params->walk_S1L2, params->walk_S1L3,
  68                   params->walk_S2L0, params->walk_S2L1,
  69                   params->walk_S2L2, params->walk_S2L3 } },
  70               params->walk_assoc, params->walk_policy),
  71     tlbEnable(params->tlb_enable),
  72     configCacheEnable(params->cfg_enable),
  73     ipaCacheEnable(params->ipa_enable),
  74     walkCacheEnable(params->walk_enable),
  75     tableWalkPortEnable(false),
  76     walkCacheNonfinalEnable(params->wc_nonfinal_enable),
  77     walkCacheS1Levels(params->wc_s1_levels),
  78     walkCacheS2Levels(params->wc_s2_levels),
  79     masterPortWidth(params->master_port_width),
  80     tlbSem(params->tlb_slots),
  81     ifcSmmuSem(1),
  82     smmuIfcSem(1),
  83     configSem(params->cfg_slots),
  84     ipaSem(params->ipa_slots),
  85     walkSem(params->walk_slots),
  86     masterPortSem(1),
  87     transSem(params->xlate_slots),
  88     ptwSem(params->ptw_slots),
  89     cycleSem(1),
  90     tlbLat(params->tlb_lat),
  91     ifcSmmuLat(params->ifc_smmu_lat),
  92     smmuIfcLat(params->smmu_ifc_lat),
  93     configLat(params->cfg_lat),
  94     ipaLat(params->ipa_lat),
  95     walkLat(params->walk_lat),
  96     slaveInterfaces(params->slave_interfaces),
  97     commandExecutor(name() + ".cmd_exec", *this),
  98     regsMap(params->reg_map),
  99     processCommandsEvent(this)
 100 {
 101     fatal_if(regsMap.size() != SMMU_REG_SIZE,
 102         "Invalid register map size: %#x different than SMMU_REG_SIZE = %#x\n",
 103         regsMap.size(), SMMU_REG_SIZE);
 104
 105     // Init smmu registers to 0
 106     memset(&regs, 0, sizeof(regs));
 107
 108     // Setup RO ID registers
 109     regs.idr0 = params->smmu_idr0;
 110     regs.idr1 = params->smmu_idr1;
 111     regs.idr2 = params->smmu_idr2;
 112     regs.idr3 = params->smmu_idr3;
 113     regs.idr4 = params->smmu_idr4;
 114     regs.idr5 = params->smmu_idr5;
 115     regs.iidr = params->smmu_iidr;
 116     regs.aidr = params->smmu_aidr;
 117
 118     // TODO: At the moment it possible to set the ID registers to hold
 119     // any possible value. It would be nice to have a sanity check here
 120     // at construction time in case some idx registers are programmed to
 121     // store an unallowed values or if the are configuration conflicts.
 122     warn("SMMUv3 IDx register values unchecked\n");
 123
 124     for (auto ifc : slaveInterfaces)
 125         ifc->setSMMU(this);
 126 }
 127
 128 bool
 129 SMMUv3::masterRecvTimingResp(PacketPtr pkt)
 130 {
 131     DPRINTF(SMMUv3, "[t] master resp addr=%#x size=%#x\n",
 132         pkt->getAddr(), pkt->getSize());
 133
 134     // @todo: We need to pay for this and not just zero it out
 135     pkt->headerDelay = pkt->payloadDelay = 0;
 136
 137     SMMUProcess *proc =
 138         safe_cast<SMMUProcess *>(pkt->popSenderState());
 139
 140     runProcessTiming(proc, pkt);
 141
 142     return true;
 143 }
 144
 145 void
 146 SMMUv3::masterRecvReqRetry()
 147 {
 148     assert(!packetsToRetry.empty());
 149
 150     while (!packetsToRetry.empty()) {
 151         SMMUAction a = packetsToRetry.front();
 152
 153         assert(a.type==ACTION_SEND_REQ || a.type==ACTION_SEND_REQ_FINAL);
 154
 155         DPRINTF(SMMUv3, "[t] master retr addr=%#x size=%#x\n",
 156             a.pkt->getAddr(), a.pkt->getSize());
 157
 158         if (!masterPort.sendTimingReq(a.pkt))
 159             break;
 160
 161         packetsToRetry.pop();
 162
 163         /*
 164          * ACTION_SEND_REQ_FINAL means that we have just forwarded the packet
 165          * on the master interface; this means that we no longer hold on to
 166          * that transaction and therefore can accept a new one.
 167          * If the slave port was stalled then unstall it (send retry).
 168          */
 169         if (a.type == ACTION_SEND_REQ_FINAL)
 170             scheduleSlaveRetries();
 171     }
 172 }
 173
 174 bool
 175 SMMUv3::masterTableWalkRecvTimingResp(PacketPtr pkt)
 176 {
 177     DPRINTF(SMMUv3, "[t] master HWTW resp addr=%#x size=%#x\n",
 178         pkt->getAddr(), pkt->getSize());
 179
 180     // @todo: We need to pay for this and not just zero it out
 181     pkt->headerDelay = pkt->payloadDelay = 0;
 182
 183     SMMUProcess *proc =
 184         safe_cast<SMMUProcess *>(pkt->popSenderState());
 185
 186     runProcessTiming(proc, pkt);
 187
 188     return true;
 189 }
 190
 191 void
 192 SMMUv3::masterTableWalkRecvReqRetry()
 193 {
 194     assert(tableWalkPortEnable);
 195     assert(!packetsTableWalkToRetry.empty());
 196
 197     while (!packetsTableWalkToRetry.empty()) {
 198         SMMUAction a = packetsTableWalkToRetry.front();
 199
 200         assert(a.type==ACTION_SEND_REQ);
 201
 202         DPRINTF(SMMUv3, "[t] master HWTW retr addr=%#x size=%#x\n",
 203             a.pkt->getAddr(), a.pkt->getSize());
 204
 205         if (!masterTableWalkPort.sendTimingReq(a.pkt))
 206             break;
 207
 208         packetsTableWalkToRetry.pop();
 209     }
 210 }
 211
 212 void
 213 SMMUv3::scheduleSlaveRetries()
 214 {
 215     for (auto ifc : slaveInterfaces) {
 216         ifc->scheduleDeviceRetry();
 217     }
 218 }
 219
 220 SMMUAction
 221 SMMUv3::runProcess(SMMUProcess *proc, PacketPtr pkt)
 222 {
 223     if (system.isAtomicMode()) {
 224         return runProcessAtomic(proc, pkt);
 225     } else if (system.isTimingMode()) {
 226         return runProcessTiming(proc, pkt);
 227     } else {
 228         panic("Not in timing or atomic mode!");
 229     }
 230 }
 231
 232 SMMUAction
 233 SMMUv3::runProcessAtomic(SMMUProcess *proc, PacketPtr pkt)
 234 {
 235     SMMUAction action;
 236     Tick delay = 0;
 237     bool finished = false;
 238
 239     do {
 240         action = proc->run(pkt);
 241
 242         switch (action.type) {
 243             case ACTION_SEND_REQ:
 244                 // Send an MMU initiated request on the table walk port if it is
 245                 // enabled. Otherwise, fall through and handle same as the final
 246                 // ACTION_SEND_REQ_FINAL request.
 247                 if (tableWalkPortEnable) {
 248                     delay += masterTableWalkPort.sendAtomic(action.pkt);
 249                     pkt = action.pkt;
 250                     break;
 251                 }
 252                 M5_FALLTHROUGH;
 253             case ACTION_SEND_REQ_FINAL:
 254                 delay += masterPort.sendAtomic(action.pkt);
 255                 pkt = action.pkt;
 256                 break;
 257
 258             case ACTION_SEND_RESP:
 259             case ACTION_SEND_RESP_ATS:
 260             case ACTION_SLEEP:
 261                 finished = true;
 262                 break;
 263
 264             case ACTION_DELAY:
 265                 delay += action.delay;
 266                 break;
 267
 268             case ACTION_TERMINATE:
 269                 panic("ACTION_TERMINATE in atomic mode\n");
 270
 271             default:
 272                 panic("Unknown action\n");
 273         }
 274     } while (!finished);
 275
 276     action.delay = delay;
 277
 278     return action;
 279 }
 280
 281 SMMUAction
 282 SMMUv3::runProcessTiming(SMMUProcess *proc, PacketPtr pkt)
 283 {
 284     SMMUAction action = proc->run(pkt);
 285
 286     switch (action.type) {
 287         case ACTION_SEND_REQ:
 288             // Send an MMU initiated request on the table walk port if it is
 289             // enabled. Otherwise, fall through and handle same as the final
 290             // ACTION_SEND_REQ_FINAL request.
 291             if (tableWalkPortEnable) {
 292                 action.pkt->pushSenderState(proc);
 293
 294                 DPRINTF(SMMUv3, "[t] master HWTW req  addr=%#x size=%#x\n",
 295                         action.pkt->getAddr(), action.pkt->getSize());
 296
 297                 if (packetsTableWalkToRetry.empty()
 298                         && masterTableWalkPort.sendTimingReq(action.pkt)) {
 299                     scheduleSlaveRetries();
 300                 } else {
 301                     DPRINTF(SMMUv3, "[t] master HWTW req  needs retry,"
 302                             " qlen=%d\n", packetsTableWalkToRetry.size());
 303                     packetsTableWalkToRetry.push(action);
 304                 }
 305
 306                 break;
 307             }
 308             M5_FALLTHROUGH;
 309         case ACTION_SEND_REQ_FINAL:
 310             action.pkt->pushSenderState(proc);
 311
 312             DPRINTF(SMMUv3, "[t] master req  addr=%#x size=%#x\n",
 313                     action.pkt->getAddr(), action.pkt->getSize());
 314
 315             if (packetsToRetry.empty() && masterPort.sendTimingReq(action.pkt)) {
 316                 scheduleSlaveRetries();
 317             } else {
 318                 DPRINTF(SMMUv3, "[t] master req  needs retry, qlen=%d\n",
 319                         packetsToRetry.size());
 320                 packetsToRetry.push(action);
 321             }
 322
 323             break;
 324
 325         case ACTION_SEND_RESP:
 326             // @todo: We need to pay for this and not just zero it out
 327             action.pkt->headerDelay = action.pkt->payloadDelay = 0;
 328
 329             DPRINTF(SMMUv3, "[t] slave resp addr=%#x size=%#x\n",
 330                     action.pkt->getAddr(),
 331                     action.pkt->getSize());
 332
 333             assert(action.ifc);
 334             action.ifc->schedTimingResp(action.pkt);
 335
 336             delete proc;
 337             break;
 338
 339         case ACTION_SEND_RESP_ATS:
 340             // @todo: We need to pay for this and not just zero it out
 341             action.pkt->headerDelay = action.pkt->payloadDelay = 0;
 342
 343             DPRINTF(SMMUv3, "[t] ATS slave resp addr=%#x size=%#x\n",
 344                     action.pkt->getAddr(), action.pkt->getSize());
 345
 346             assert(action.ifc);
 347             action.ifc->schedAtsTimingResp(action.pkt);
 348
 349             delete proc;
 350             break;
 351
 352         case ACTION_DELAY:
 353         case ACTION_SLEEP:
 354             break;
 355
 356         case ACTION_TERMINATE:
 357             delete proc;
 358             break;
 359
 360         default:
 361             panic("Unknown action\n");
 362     }
 363
 364     return action;
 365 }
 366
 367 void
 368 SMMUv3::processCommands()
 369 {
 370     DPRINTF(SMMUv3, "processCommands()\n");
 371
 372     if (system.isAtomicMode()) {
 373         SMMUAction a = runProcessAtomic(&commandExecutor, NULL);
 374         (void) a;
 375     } else if (system.isTimingMode()) {
 376         if (!commandExecutor.isBusy())
 377             runProcessTiming(&commandExecutor, NULL);
 378     } else {
 379         panic("Not in timing or atomic mode!");
 380     }
 381 }
 382
 383 void
 384 SMMUv3::processCommand(const SMMUCommand &cmd)
 385 {
 386     switch (cmd.type) {
 387         case CMD_PRF_CONFIG:
 388             DPRINTF(SMMUv3, "CMD_PREFETCH_CONFIG - ignored\n");
 389             break;
 390
 391         case CMD_PRF_ADDR:
 392             DPRINTF(SMMUv3, "CMD_PREFETCH_ADDR - ignored\n");
 393             break;
 394
 395         case CMD_INV_STE:
 396             DPRINTF(SMMUv3, "CMD_INV_STE sid=%#x\n", cmd.data[0]);
 397             configCache.invalidateSID(cmd.data[0]);
 398             break;
 399
 400         case CMD_INV_CD:
 401             DPRINTF(SMMUv3, "CMD_INV_CD sid=%#x ssid=%#x\n",
 402                 cmd.data[0], cmd.data[1]);
 403             configCache.invalidateSSID(cmd.data[0], cmd.data[1]);
 404             break;
 405
 406         case CMD_INV_CD_ALL:
 407             DPRINTF(SMMUv3, "CMD_INV_CD_ALL sid=%#x\n", cmd.data[0]);
 408             configCache.invalidateSID(cmd.data[0]);
 409             break;
 410
 411         case CMD_INV_ALL:
 412             DPRINTF(SMMUv3, "CMD_INV_ALL\n");
 413             configCache.invalidateAll();
 414             break;
 415
 416         case CMD_TLBI_ALL:
 417             DPRINTF(SMMUv3, "CMD_TLBI_ALL\n");
 418             for (auto slave_interface : slaveInterfaces) {
 419                 slave_interface->microTLB->invalidateAll();
 420                 slave_interface->mainTLB->invalidateAll();
 421             }
 422             tlb.invalidateAll();
 423             ipaCache.invalidateAll();
 424             walkCache.invalidateAll();
 425             break;
 426
 427         case CMD_TLBI_ASID:
 428             DPRINTF(SMMUv3, "CMD_TLBI_ASID asid=%#x vmid=%#x\n",
 429                 cmd.data[0], cmd.data[1]);
 430             for (auto slave_interface : slaveInterfaces) {
 431                 slave_interface->microTLB->invalidateASID(
 432                     cmd.data[0], cmd.data[1]);
 433                 slave_interface->mainTLB->invalidateASID(
 434                     cmd.data[0], cmd.data[1]);
 435             }
 436             tlb.invalidateASID(cmd.data[0], cmd.data[1]);
 437             walkCache.invalidateASID(cmd.data[0], cmd.data[1]);
 438             break;
 439
 440         case CMD_TLBI_VAAL:
 441             DPRINTF(SMMUv3, "CMD_TLBI_VAAL va=%#08x vmid=%#x\n",
 442                 cmd.data[0], cmd.data[1]);
 443             for (auto slave_interface : slaveInterfaces) {
 444                 slave_interface->microTLB->invalidateVAA(
 445                     cmd.data[0], cmd.data[1]);
 446                 slave_interface->mainTLB->invalidateVAA(
 447                     cmd.data[0], cmd.data[1]);
 448             }
 449             tlb.invalidateVAA(cmd.data[0], cmd.data[1]);
 450             break;
 451
 452         case CMD_TLBI_VAA:
 453             DPRINTF(SMMUv3, "CMD_TLBI_VAA va=%#08x vmid=%#x\n",
 454                 cmd.data[0], cmd.data[1]);
 455             for (auto slave_interface : slaveInterfaces) {
 456                 slave_interface->microTLB->invalidateVAA(
 457                     cmd.data[0], cmd.data[1]);
 458                 slave_interface->mainTLB->invalidateVAA(
 459                     cmd.data[0], cmd.data[1]);
 460             }
 461             tlb.invalidateVAA(cmd.data[0], cmd.data[1]);
 462             walkCache.invalidateVAA(cmd.data[0], cmd.data[1]);
 463             break;
 464
 465         case CMD_TLBI_VAL:
 466             DPRINTF(SMMUv3, "CMD_TLBI_VAL va=%#08x asid=%#x vmid=%#x\n",
 467                 cmd.data[0], cmd.data[1], cmd.data[2]);
 468             for (auto slave_interface : slaveInterfaces) {
 469                 slave_interface->microTLB->invalidateVA(
 470                     cmd.data[0], cmd.data[1], cmd.data[2]);
 471                 slave_interface->mainTLB->invalidateVA(
 472                     cmd.data[0], cmd.data[1], cmd.data[2]);
 473             }
 474             tlb.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]);
 475             break;
 476
 477         case CMD_TLBI_VA:
 478             DPRINTF(SMMUv3, "CMD_TLBI_VA va=%#08x asid=%#x vmid=%#x\n",
 479                 cmd.data[0], cmd.data[1], cmd.data[2]);
 480             for (auto slave_interface : slaveInterfaces) {
 481                 slave_interface->microTLB->invalidateVA(
 482                     cmd.data[0], cmd.data[1], cmd.data[2]);
 483                 slave_interface->mainTLB->invalidateVA(
 484                     cmd.data[0], cmd.data[1], cmd.data[2]);
 485             }
 486             tlb.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]);
 487             walkCache.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]);
 488             break;
 489
 490         case CMD_TLBI_VM_IPAL:
 491             DPRINTF(SMMUv3, "CMD_TLBI_VM_IPAL ipa=%#08x vmid=%#x\n",
 492                 cmd.data[0], cmd.data[1]);
 493             // This does not invalidate TLBs containing
 494             // combined Stage1 + Stage2 translations, as per the spec.
 495             ipaCache.invalidateIPA(cmd.data[0], cmd.data[1]);
 496             walkCache.invalidateVMID(cmd.data[1]);
 497             break;
 498
 499         case CMD_TLBI_VM_IPA:
 500             DPRINTF(SMMUv3, "CMD_TLBI_VM_IPA ipa=%#08x vmid=%#x\n",
 501                 cmd.data[0], cmd.data[1]);
 502             // This does not invalidate TLBs containing
 503             // combined Stage1 + Stage2 translations, as per the spec.
 504             ipaCache.invalidateIPA(cmd.data[0], cmd.data[1]);
 505             walkCache.invalidateVMID(cmd.data[1]);
 506             break;
 507
 508         case CMD_TLBI_VM_S12:
 509             DPRINTF(SMMUv3, "CMD_TLBI_VM_S12 vmid=%#x\n", cmd.data[0]);
 510             for (auto slave_interface : slaveInterfaces) {
 511                 slave_interface->microTLB->invalidateVMID(cmd.data[0]);
 512                 slave_interface->mainTLB->invalidateVMID(cmd.data[0]);
 513             }
 514             tlb.invalidateVMID(cmd.data[0]);
 515             ipaCache.invalidateVMID(cmd.data[0]);
 516             walkCache.invalidateVMID(cmd.data[0]);
 517             break;
 518
 519         case CMD_RESUME_S:
 520             DPRINTF(SMMUv3, "CMD_RESUME_S\n");
 521             panic("resume unimplemented");
 522             break;
 523
 524         default:
 525             warn("Unimplemented command %#x\n", cmd.type);
 526             break;
 527     }
 528 }
 529
 530 const PageTableOps*
 531 SMMUv3::getPageTableOps(uint8_t trans_granule)
 532 {
 533     static V8PageTableOps4k  ptOps4k;
 534     static V8PageTableOps16k ptOps16k;
 535     static V8PageTableOps64k ptOps64k;
 536
 537     switch (trans_granule) {
 538     case TRANS_GRANULE_4K:  return &ptOps4k;
 539     case TRANS_GRANULE_16K: return &ptOps16k;
 540     case TRANS_GRANULE_64K: return &ptOps64k;
 541     default:
 542         panic("Unknown translation granule size %d", trans_granule);
 543     }
 544 }
 545
 546 Tick
 547 SMMUv3::readControl(PacketPtr pkt)
 548 {
 549     DPRINTF(SMMUv3, "readControl:  addr=%08x size=%d\n",
 550             pkt->getAddr(), pkt->getSize());
 551
 552     int offset = pkt->getAddr() - regsMap.start();
 553     assert(offset >= 0 && offset < SMMU_REG_SIZE);
 554
 555     if (inSecureBlock(offset)) {
 556         warn("smmu: secure registers (0x%x) are not implemented\n",
 557              offset);
 558     }
 559
 560     auto reg_ptr = regs.data + offset;
 561
 562     switch (pkt->getSize()) {
 563       case sizeof(uint32_t):
 564         pkt->setLE<uint32_t>(*reinterpret_cast<uint32_t *>(reg_ptr));
 565         break;
 566       case sizeof(uint64_t):
 567         pkt->setLE<uint64_t>(*reinterpret_cast<uint64_t *>(reg_ptr));
 568         break;
 569       default:
 570         panic("smmu: unallowed access size: %d bytes\n", pkt->getSize());
 571         break;
 572     }
 573
 574     pkt->makeAtomicResponse();
 575
 576     return 0;
 577 }
 578
 579 Tick
 580 SMMUv3::writeControl(PacketPtr pkt)
 581 {
 582     int offset = pkt->getAddr() - regsMap.start();
 583     assert(offset >= 0 && offset < SMMU_REG_SIZE);
 584
 585     DPRINTF(SMMUv3, "writeControl: addr=%08x size=%d data=%16x\n",
 586             pkt->getAddr(), pkt->getSize(),
 587             pkt->getSize() == sizeof(uint64_t) ?
 588             pkt->getLE<uint64_t>() : pkt->getLE<uint32_t>());
 589
 590     switch (offset) {
 591         case offsetof(SMMURegs, cr0):
 592             assert(pkt->getSize() == sizeof(uint32_t));
 593             regs.cr0 = regs.cr0ack = pkt->getLE<uint32_t>();
 594             break;
 595
 596         case offsetof(SMMURegs, cr1):
 597         case offsetof(SMMURegs, cr2):
 598         case offsetof(SMMURegs, strtab_base_cfg):
 599         case offsetof(SMMURegs, eventq_cons):
 600         case offsetof(SMMURegs, eventq_irq_cfg1):
 601         case offsetof(SMMURegs, priq_cons):
 602             assert(pkt->getSize() == sizeof(uint32_t));
 603             *reinterpret_cast<uint32_t *>(regs.data + offset) =
 604                 pkt->getLE<uint32_t>();
 605             break;
 606
 607         case offsetof(SMMURegs, cmdq_cons):
 608             assert(pkt->getSize() == sizeof(uint32_t));
 609             if (regs.cr0 & CR0_CMDQEN_MASK) {
 610                 warn("CMDQ is enabled: ignoring write to CMDQ_CONS\n");
 611             } else {
 612                 *reinterpret_cast<uint32_t *>(regs.data + offset) =
 613                     pkt->getLE<uint32_t>();
 614             }
 615             break;
 616
 617         case offsetof(SMMURegs, cmdq_prod):
 618             assert(pkt->getSize() == sizeof(uint32_t));
 619             *reinterpret_cast<uint32_t *>(regs.data + offset) =
 620                 pkt->getLE<uint32_t>();
 621             schedule(processCommandsEvent, nextCycle());
 622             break;
 623
 624         case offsetof(SMMURegs, strtab_base):
 625         case offsetof(SMMURegs, eventq_irq_cfg0):
 626             assert(pkt->getSize() == sizeof(uint64_t));
 627             *reinterpret_cast<uint64_t *>(regs.data + offset) =
 628                 pkt->getLE<uint64_t>();
 629             break;
 630
 631         case offsetof(SMMURegs, cmdq_base):
 632             assert(pkt->getSize() == sizeof(uint64_t));
 633             if (regs.cr0 & CR0_CMDQEN_MASK) {
 634                 warn("CMDQ is enabled: ignoring write to CMDQ_BASE\n");
 635             } else {
 636                 *reinterpret_cast<uint64_t *>(regs.data + offset) =
 637                     pkt->getLE<uint64_t>();
 638                 regs.cmdq_cons = 0;
 639                 regs.cmdq_prod = 0;
 640             }
 641             break;
 642
 643         case offsetof(SMMURegs, eventq_base):
 644             assert(pkt->getSize() == sizeof(uint64_t));
 645             *reinterpret_cast<uint64_t *>(regs.data + offset) =
 646                 pkt->getLE<uint64_t>();
 647             regs.eventq_cons = 0;
 648             regs.eventq_prod = 0;
 649             break;
 650
 651         case offsetof(SMMURegs, priq_base):
 652             assert(pkt->getSize() == sizeof(uint64_t));
 653             *reinterpret_cast<uint64_t *>(regs.data + offset) =
 654                 pkt->getLE<uint64_t>();
 655             regs.priq_cons = 0;
 656             regs.priq_prod = 0;
 657             break;
 658
 659         default:
 660             if (inSecureBlock(offset)) {
 661                 warn("smmu: secure registers (0x%x) are not implemented\n",
 662                      offset);
 663             } else {
 664                 warn("smmu: write to read-only/undefined register at 0x%x\n",
 665                      offset);
 666             }
 667     }
 668
 669     pkt->makeAtomicResponse();
 670
 671     return 0;
 672 }
 673
 674 bool
 675 SMMUv3::inSecureBlock(uint32_t offs) const
 676 {
 677     if (offs >= offsetof(SMMURegs, _secure_regs) && offs < SMMU_SECURE_SZ)
 678         return true;
 679     else
 680         return false;
 681 }
 682
 683 void
 684 SMMUv3::init()
 685 {
 686     // make sure both sides are connected and have the same block size
 687     if (!masterPort.isConnected())
 688         fatal("Master port is not connected.\n");
 689
 690     // If the second master port is connected for the table walks, enable
 691     // the mode to send table walks through this port instead
 692     if (masterTableWalkPort.isConnected())
 693         tableWalkPortEnable = true;
 694
 695     // notify the master side  of our address ranges
 696     for (auto ifc : slaveInterfaces) {
 697         ifc->sendRange();
 698     }
 699
 700     if (controlPort.isConnected())
 701         controlPort.sendRangeChange();
 702 }
 703
 704 void
 705 SMMUv3::regStats()
 706 {
 707     MemObject::regStats();
 708
 709     using namespace Stats;
 710
 711     for (size_t i = 0; i < slaveInterfaces.size(); i++) {
 712         slaveInterfaces[i]->microTLB->regStats(
 713             csprintf("%s.utlb%d", name(), i));
 714         slaveInterfaces[i]->mainTLB->regStats(
 715             csprintf("%s.maintlb%d", name(), i));
 716     }
 717
 718     tlb.regStats(name() + ".tlb");
 719     configCache.regStats(name() + ".cfg");
 720     ipaCache.regStats(name() + ".ipa");
 721     walkCache.regStats(name() + ".walk");
 722
 723     steL1Fetches
 724         .name(name() + ".steL1Fetches")
 725         .desc("STE L1 fetches")
 726         .flags(pdf);
 727
 728     steFetches
 729         .name(name() + ".steFetches")
 730         .desc("STE fetches")
 731         .flags(pdf);
 732
 733     cdL1Fetches
 734         .name(name() + ".cdL1Fetches")
 735         .desc("CD L1 fetches")
 736         .flags(pdf);
 737
 738     cdFetches
 739         .name(name() + ".cdFetches")
 740         .desc("CD fetches")
 741         .flags(pdf);
 742
 743     translationTimeDist
 744         .init(0, 2000000, 2000)
 745         .name(name() + ".translationTimeDist")
 746         .desc("Time to translate address")
 747         .flags(pdf);
 748
 749     ptwTimeDist
 750         .init(0, 2000000, 2000)
 751         .name(name() + ".ptwTimeDist")
 752         .desc("Time to walk page tables")
 753         .flags(pdf);
 754 }
 755
 756 DrainState
 757 SMMUv3::drain()
 758 {
 759     // Wait until the Command Executor is not busy
 760     if (commandExecutor.isBusy()) {
 761         return DrainState::Draining;
 762     }
 763     return DrainState::Drained;
 764 }
 765
 766 void
 767 SMMUv3::serialize(CheckpointOut &cp) const
 768 {
 769     DPRINTF(Checkpoint, "Serializing SMMUv3\n");
 770
 771     SERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0]));
 772 }
 773
 774 void
 775 SMMUv3::unserialize(CheckpointIn &cp)
 776 {
 777     DPRINTF(Checkpoint, "Unserializing SMMUv3\n");
 778
 779     UNSERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0]));
 780 }
 781
 782 Port&
 783 SMMUv3::getPort(const std::string &name, PortID id)
 784 {
 785     if (name == "master") {
 786         return masterPort;
 787     } else if (name == "master_walker") {
 788         return masterTableWalkPort;
 789     } else if (name == "control") {
 790         return controlPort;
 791     } else {
 792         return MemObject::getPort(name, id);
 793     }
 794 }
 795
 796 SMMUv3*
 797 SMMUv3Params::create()
 798 {
 799     return new SMMUv3(this);
 800 }