src/arch/arm/table_walker.cc

   1 /*
   2  * Copyright (c) 2010, 2012-2015 ARM Limited
   3  * All rights reserved
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  *
  37  * Authors: Ali Saidi
  38  *          Giacomo Gabrielli
  39  */
  40
  41 #include <memory>
  42
  43 #include "arch/arm/faults.hh"
  44 #include "arch/arm/stage2_mmu.hh"
  45 #include "arch/arm/system.hh"
  46 #include "arch/arm/table_walker.hh"
  47 #include "arch/arm/tlb.hh"
  48 #include "cpu/base.hh"
  49 #include "cpu/thread_context.hh"
  50 #include "debug/Checkpoint.hh"
  51 #include "debug/Drain.hh"
  52 #include "debug/TLB.hh"
  53 #include "debug/TLBVerbose.hh"
  54 #include "sim/system.hh"
  55
  56 using namespace ArmISA;
  57
  58 TableWalker::TableWalker(const Params *p)
  59     : MemObject(p), drainManager(NULL),
  60       stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId),
  61       isStage2(p->is_stage2), tlb(NULL),
  62       currState(NULL), pending(false),
  63       numSquashable(p->num_squash_per_cycle),
  64       pendingReqs(0),
  65       pendingChangeTick(curTick()),
  66       doL1DescEvent(this), doL2DescEvent(this),
  67       doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this),
  68       doL3LongDescEvent(this),
  69       doProcessEvent(this)
  70 {
  71     sctlr = 0;
  72
  73     // Cache system-level properties
  74     if (FullSystem) {
  75         ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys);
  76         assert(armSys);
  77         haveSecurity = armSys->haveSecurity();
  78         _haveLPAE = armSys->haveLPAE();
  79         _haveVirtualization = armSys->haveVirtualization();
  80         physAddrRange = armSys->physAddrRange();
  81         _haveLargeAsid64 = armSys->haveLargeAsid64();
  82     } else {
  83         haveSecurity = _haveLPAE = _haveVirtualization = false;
  84         _haveLargeAsid64 = false;
  85         physAddrRange = 32;
  86     }
  87
  88 }
  89
  90 TableWalker::~TableWalker()
  91 {
  92     ;
  93 }
  94
  95 void
  96 TableWalker::setMMU(Stage2MMU *m, MasterID master_id)
  97 {
  98     stage2Mmu = m;
  99     port = &m->getPort();
 100     masterId = master_id;
 101 }
 102
 103 void
 104 TableWalker::init()
 105 {
 106     fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n");
 107     fatal_if(!port, "Table walker must have a valid port\n");
 108     fatal_if(!tlb, "Table walker must have a valid TLB\n");
 109 }
 110
 111 BaseMasterPort&
 112 TableWalker::getMasterPort(const std::string &if_name, PortID idx)
 113 {
 114     if (if_name == "port") {
 115         if (!isStage2) {
 116             return *port;
 117         } else {
 118             fatal("Cannot access table walker port through stage-two walker\n");
 119         }
 120     }
 121     return MemObject::getMasterPort(if_name, idx);
 122 }
 123
 124 TableWalker::WalkerState::WalkerState() :
 125     tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr),
 126     asid(0), vmid(0), isHyp(false), transState(nullptr),
 127     vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false),
 128     secureLookup(false), rwTable(false), userTable(false), xnTable(false),
 129     pxnTable(false), stage2Req(false), doingStage2(false),
 130     stage2Tran(nullptr), timing(false), functional(false),
 131     mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc),
 132     delayed(false), tableWalker(nullptr)
 133 {
 134 }
 135
 136 void
 137 TableWalker::completeDrain()
 138 {
 139     if (drainManager && stateQueues[L1].empty() && stateQueues[L2].empty() &&
 140         pendingQueue.empty()) {
 141         setDrainState(Drainable::Drained);
 142         DPRINTF(Drain, "TableWalker done draining, processing drain event\n");
 143         drainManager->signalDrainDone();
 144         drainManager = NULL;
 145     }
 146 }
 147
 148 unsigned int
 149 TableWalker::drain(DrainManager *dm)
 150 {
 151     bool state_queues_not_empty = false;
 152
 153     for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) {
 154         if (!stateQueues[i].empty()) {
 155             state_queues_not_empty = true;
 156             break;
 157         }
 158     }
 159
 160     if (state_queues_not_empty || pendingQueue.size()) {
 161         drainManager = dm;
 162         setDrainState(Drainable::Draining);
 163         DPRINTF(Drain, "TableWalker not drained\n");
 164
 165         // return port drain count plus the table walker itself needs to drain
 166         return 1;
 167     } else {
 168         setDrainState(Drainable::Drained);
 169         DPRINTF(Drain, "TableWalker free, no need to drain\n");
 170
 171         // table walker is drained, but its ports may still need to be drained
 172         return 0;
 173     }
 174 }
 175
 176 void
 177 TableWalker::drainResume()
 178 {
 179     Drainable::drainResume();
 180     if (params()->sys->isTimingMode() && currState) {
 181         delete currState;
 182         currState = NULL;
 183         pendingChange();
 184     }
 185 }
 186
 187 Fault
 188 TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
 189                   uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
 190                   TLB::Translation *_trans, bool _timing, bool _functional,
 191                   bool secure, TLB::ArmTranslationType tranType)
 192 {
 193     assert(!(_functional && _timing));
 194     ++statWalks;
 195
 196     WalkerState *savedCurrState = NULL;
 197
 198     if (!currState && !_functional) {
 199         // For atomic mode, a new WalkerState instance should be only created
 200         // once per TLB. For timing mode, a new instance is generated for every
 201         // TLB miss.
 202         DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
 203
 204         currState = new WalkerState();
 205         currState->tableWalker = this;
 206     } else if (_functional) {
 207         // If we are mixing functional mode with timing (or even
 208         // atomic), we need to to be careful and clean up after
 209         // ourselves to not risk getting into an inconsistent state.
 210         DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
 211         savedCurrState = currState;
 212         currState = new WalkerState();
 213         currState->tableWalker = this;
 214     } else if (_timing) {
 215         // This is a translation that was completed and then faulted again
 216         // because some underlying parameters that affect the translation
 217         // changed out from under us (e.g. asid). It will either be a
 218         // misprediction, in which case nothing will happen or we'll use
 219         // this fault to re-execute the faulting instruction which should clean
 220         // up everything.
 221         if (currState->vaddr_tainted == _req->getVaddr()) {
 222             ++statSquashedBefore;
 223             return std::make_shared<ReExec>();
 224         }
 225     }
 226     pendingChange();
 227
 228     currState->startTime = curTick();
 229     currState->tc = _tc;
 230     currState->aarch64 = opModeIs64(currOpMode(_tc));
 231     currState->el = currEL(_tc);
 232     currState->transState = _trans;
 233     currState->req = _req;
 234     currState->fault = NoFault;
 235     currState->asid = _asid;
 236     currState->vmid = _vmid;
 237     currState->isHyp = _isHyp;
 238     currState->timing = _timing;
 239     currState->functional = _functional;
 240     currState->mode = _mode;
 241     currState->tranType = tranType;
 242     currState->isSecure = secure;
 243     currState->physAddrRange = physAddrRange;
 244
 245     /** @todo These should be cached or grabbed from cached copies in
 246      the TLB, all these miscreg reads are expensive */
 247     currState->vaddr_tainted = currState->req->getVaddr();
 248     if (currState->aarch64)
 249         currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
 250                                             currState->tc, currState->el);
 251     else
 252         currState->vaddr = currState->vaddr_tainted;
 253
 254     if (currState->aarch64) {
 255         switch (currState->el) {
 256           case EL0:
 257           case EL1:
 258             currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
 259             currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
 260             break;
 261           // @todo: uncomment this to enable Virtualization
 262           // case EL2:
 263           //   assert(haveVirtualization);
 264           //   currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
 265           //   currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
 266           //   break;
 267           case EL3:
 268             assert(haveSecurity);
 269             currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
 270             currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
 271             break;
 272           default:
 273             panic("Invalid exception level");
 274             break;
 275         }
 276     } else {
 277         currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
 278             MISCREG_SCTLR, currState->tc, !currState->isSecure));
 279         currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
 280             MISCREG_TTBCR, currState->tc, !currState->isSecure));
 281         currState->htcr  = currState->tc->readMiscReg(MISCREG_HTCR);
 282         currState->hcr   = currState->tc->readMiscReg(MISCREG_HCR);
 283         currState->vtcr  = currState->tc->readMiscReg(MISCREG_VTCR);
 284     }
 285     sctlr = currState->sctlr;
 286
 287     currState->isFetch = (currState->mode == TLB::Execute);
 288     currState->isWrite = (currState->mode == TLB::Write);
 289
 290     statRequestOrigin[REQUESTED][currState->isFetch]++;
 291
 292     // We only do a second stage of translation if we're not secure, or in
 293     // hyp mode, the second stage MMU is enabled, and this table walker
 294     // instance is the first stage.
 295     currState->doingStage2 = false;
 296     // @todo: for now disable this in AArch64 (HCR is not set)
 297     currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
 298                            !isStage2 && !currState->isSecure && !currState->isHyp;
 299
 300     bool long_desc_format = currState->aarch64 ||
 301                             (_haveLPAE && currState->ttbcr.eae) ||
 302                             _isHyp || isStage2;
 303
 304     if (long_desc_format) {
 305         // Helper variables used for hierarchical permissions
 306         currState->secureLookup = currState->isSecure;
 307         currState->rwTable = true;
 308         currState->userTable = true;
 309         currState->xnTable = false;
 310         currState->pxnTable = false;
 311
 312         ++statWalksLongDescriptor;
 313     } else {
 314         ++statWalksShortDescriptor;
 315     }
 316
 317     if (!currState->timing) {
 318         Fault fault = NoFault;
 319         if (currState->aarch64)
 320             fault = processWalkAArch64();
 321         else if (long_desc_format)
 322             fault = processWalkLPAE();
 323         else
 324             fault = processWalk();
 325
 326         // If this was a functional non-timing access restore state to
 327         // how we found it.
 328         if (currState->functional) {
 329             delete currState;
 330             currState = savedCurrState;
 331         }
 332         return fault;
 333     }
 334
 335     if (pending || pendingQueue.size()) {
 336         pendingQueue.push_back(currState);
 337         currState = NULL;
 338         pendingChange();
 339     } else {
 340         pending = true;
 341         pendingChange();
 342         if (currState->aarch64)
 343             return processWalkAArch64();
 344         else if (long_desc_format)
 345             return processWalkLPAE();
 346         else
 347             return processWalk();
 348     }
 349
 350     return NoFault;
 351 }
 352
 353 void
 354 TableWalker::processWalkWrapper()
 355 {
 356     assert(!currState);
 357     assert(pendingQueue.size());
 358     pendingChange();
 359     currState = pendingQueue.front();
 360
 361     ExceptionLevel target_el = EL0;
 362     if (currState->aarch64)
 363         target_el = currEL(currState->tc);
 364     else
 365         target_el = EL1;
 366
 367     // Check if a previous walk filled this request already
 368     // @TODO Should this always be the TLB or should we look in the stage2 TLB?
 369     TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
 370             currState->vmid, currState->isHyp, currState->isSecure, true, false,
 371             target_el);
 372
 373     // Check if we still need to have a walk for this request. If the requesting
 374     // instruction has been squashed, or a previous walk has filled the TLB with
 375     // a match, we just want to get rid of the walk. The latter could happen
 376     // when there are multiple outstanding misses to a single page and a
 377     // previous request has been successfully translated.
 378     if (!currState->transState->squashed() && !te) {
 379         // We've got a valid request, lets process it
 380         pending = true;
 381         pendingQueue.pop_front();
 382         // Keep currState in case one of the processWalk... calls NULLs it
 383         WalkerState *curr_state_copy = currState;
 384         Fault f;
 385         if (currState->aarch64)
 386             f = processWalkAArch64();
 387         else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
 388             f = processWalkLPAE();
 389         else
 390             f = processWalk();
 391
 392         if (f != NoFault) {
 393             curr_state_copy->transState->finish(f, curr_state_copy->req,
 394                     curr_state_copy->tc, curr_state_copy->mode);
 395
 396             delete curr_state_copy;
 397         }
 398         return;
 399     }
 400
 401
 402     // If the instruction that we were translating for has been
 403     // squashed we shouldn't bother.
 404     unsigned num_squashed = 0;
 405     ThreadContext *tc = currState->tc;
 406     while ((num_squashed < numSquashable) && currState &&
 407            (currState->transState->squashed() || te)) {
 408         pendingQueue.pop_front();
 409         num_squashed++;
 410         statSquashedBefore++;
 411
 412         DPRINTF(TLB, "Squashing table walk for address %#x\n",
 413                       currState->vaddr_tainted);
 414
 415         if (currState->transState->squashed()) {
 416             // finish the translation which will delete the translation object
 417             currState->transState->finish(
 418                 std::make_shared<UnimpFault>("Squashed Inst"),
 419                 currState->req, currState->tc, currState->mode);
 420         } else {
 421             // translate the request now that we know it will work
 422             statWalkServiceTime.sample(curTick() - currState->startTime);
 423             tlb->translateTiming(currState->req, currState->tc,
 424                         currState->transState, currState->mode);
 425
 426         }
 427
 428         // delete the current request
 429         delete currState;
 430
 431         // peak at the next one
 432         if (pendingQueue.size()) {
 433             currState = pendingQueue.front();
 434             te = tlb->lookup(currState->vaddr, currState->asid,
 435                 currState->vmid, currState->isHyp, currState->isSecure, true,
 436                 false, target_el);
 437         } else {
 438             // Terminate the loop, nothing more to do
 439             currState = NULL;
 440         }
 441     }
 442     pendingChange();
 443
 444     // if we still have pending translations, schedule more work
 445     nextWalk(tc);
 446     currState = NULL;
 447 }
 448
 449 Fault
 450 TableWalker::processWalk()
 451 {
 452     Addr ttbr = 0;
 453
 454     // If translation isn't enabled, we shouldn't be here
 455     assert(currState->sctlr.m || isStage2);
 456
 457     DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
 458             currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
 459                                                       32 - currState->ttbcr.n));
 460
 461     statWalkWaitTime.sample(curTick() - currState->startTime);
 462
 463     if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
 464                                           32 - currState->ttbcr.n)) {
 465         DPRINTF(TLB, " - Selecting TTBR0\n");
 466         // Check if table walk is allowed when Security Extensions are enabled
 467         if (haveSecurity && currState->ttbcr.pd0) {
 468             if (currState->isFetch)
 469                 return std::make_shared<PrefetchAbort>(
 470                     currState->vaddr_tainted,
 471                     ArmFault::TranslationLL + L1,
 472                     isStage2,
 473                     ArmFault::VmsaTran);
 474             else
 475                 return std::make_shared<DataAbort>(
 476                     currState->vaddr_tainted,
 477                     TlbEntry::DomainType::NoAccess, currState->isWrite,
 478                     ArmFault::TranslationLL + L1, isStage2,
 479                     ArmFault::VmsaTran);
 480         }
 481         ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
 482             MISCREG_TTBR0, currState->tc, !currState->isSecure));
 483     } else {
 484         DPRINTF(TLB, " - Selecting TTBR1\n");
 485         // Check if table walk is allowed when Security Extensions are enabled
 486         if (haveSecurity && currState->ttbcr.pd1) {
 487             if (currState->isFetch)
 488                 return std::make_shared<PrefetchAbort>(
 489                     currState->vaddr_tainted,
 490                     ArmFault::TranslationLL + L1,
 491                     isStage2,
 492                     ArmFault::VmsaTran);
 493             else
 494                 return std::make_shared<DataAbort>(
 495                     currState->vaddr_tainted,
 496                     TlbEntry::DomainType::NoAccess, currState->isWrite,
 497                     ArmFault::TranslationLL + L1, isStage2,
 498                     ArmFault::VmsaTran);
 499         }
 500         ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
 501             MISCREG_TTBR1, currState->tc, !currState->isSecure));
 502         currState->ttbcr.n = 0;
 503     }
 504
 505     Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
 506         (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
 507     DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
 508             currState->isSecure ? "s" : "ns");
 509
 510     // Trickbox address check
 511     Fault f;
 512     f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
 513             currState->vaddr, sizeof(uint32_t), currState->isFetch,
 514             currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
 515     if (f) {
 516         DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
 517         if (currState->timing) {
 518             pending = false;
 519             nextWalk(currState->tc);
 520             currState = NULL;
 521         } else {
 522             currState->tc = NULL;
 523             currState->req = NULL;
 524         }
 525         return f;
 526     }
 527
 528     Request::Flags flag = 0;
 529     if (currState->sctlr.c == 0) {
 530         flag = Request::UNCACHEABLE;
 531     }
 532
 533     bool delayed;
 534     delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
 535                               sizeof(uint32_t), flag, L1, &doL1DescEvent,
 536                               &TableWalker::doL1Descriptor);
 537     if (!delayed) {
 538        f = currState->fault;
 539     }
 540
 541     return f;
 542 }
 543
 544 Fault
 545 TableWalker::processWalkLPAE()
 546 {
 547     Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
 548     int tsz, n;
 549     LookupLevel start_lookup_level = L1;
 550
 551     DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
 552             currState->vaddr_tainted, currState->ttbcr);
 553
 554     statWalkWaitTime.sample(curTick() - currState->startTime);
 555
 556     Request::Flags flag = 0;
 557     if (currState->isSecure)
 558         flag.set(Request::SECURE);
 559
 560     // work out which base address register to use, if in hyp mode we always
 561     // use HTTBR
 562     if (isStage2) {
 563         DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
 564         ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
 565         tsz  = sext<4>(currState->vtcr.t0sz);
 566         start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
 567     } else if (currState->isHyp) {
 568         DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
 569         ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
 570         tsz  = currState->htcr.t0sz;
 571     } else {
 572         assert(_haveLPAE && currState->ttbcr.eae);
 573
 574         // Determine boundaries of TTBR0/1 regions
 575         if (currState->ttbcr.t0sz)
 576             ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
 577         else if (currState->ttbcr.t1sz)
 578             ttbr0_max = (1ULL << 32) -
 579                 (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
 580         else
 581             ttbr0_max = (1ULL << 32) - 1;
 582         if (currState->ttbcr.t1sz)
 583             ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
 584         else
 585             ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
 586
 587         // The following code snippet selects the appropriate translation table base
 588         // address (TTBR0 or TTBR1) and the appropriate starting lookup level
 589         // depending on the address range supported by the translation table (ARM
 590         // ARM issue C B3.6.4)
 591         if (currState->vaddr <= ttbr0_max) {
 592             DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
 593             // Check if table walk is allowed
 594             if (currState->ttbcr.epd0) {
 595                 if (currState->isFetch)
 596                     return std::make_shared<PrefetchAbort>(
 597                         currState->vaddr_tainted,
 598                         ArmFault::TranslationLL + L1,
 599                         isStage2,
 600                         ArmFault::LpaeTran);
 601                 else
 602                     return std::make_shared<DataAbort>(
 603                         currState->vaddr_tainted,
 604                         TlbEntry::DomainType::NoAccess,
 605                         currState->isWrite,
 606                         ArmFault::TranslationLL + L1,
 607                         isStage2,
 608                         ArmFault::LpaeTran);
 609             }
 610             ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
 611                 MISCREG_TTBR0, currState->tc, !currState->isSecure));
 612             tsz = currState->ttbcr.t0sz;
 613             if (ttbr0_max < (1ULL << 30))  // Upper limit < 1 GB
 614                 start_lookup_level = L2;
 615         } else if (currState->vaddr >= ttbr1_min) {
 616             DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
 617             // Check if table walk is allowed
 618             if (currState->ttbcr.epd1) {
 619                 if (currState->isFetch)
 620                     return std::make_shared<PrefetchAbort>(
 621                         currState->vaddr_tainted,
 622                         ArmFault::TranslationLL + L1,
 623                         isStage2,
 624                         ArmFault::LpaeTran);
 625                 else
 626                     return std::make_shared<DataAbort>(
 627                         currState->vaddr_tainted,
 628                         TlbEntry::DomainType::NoAccess,
 629                         currState->isWrite,
 630                         ArmFault::TranslationLL + L1,
 631                         isStage2,
 632                         ArmFault::LpaeTran);
 633             }
 634             ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
 635                 MISCREG_TTBR1, currState->tc, !currState->isSecure));
 636             tsz = currState->ttbcr.t1sz;
 637             if (ttbr1_min >= (1ULL << 31) + (1ULL << 30))  // Lower limit >= 3 GB
 638                 start_lookup_level = L2;
 639         } else {
 640             // Out of boundaries -> translation fault
 641             if (currState->isFetch)
 642                 return std::make_shared<PrefetchAbort>(
 643                     currState->vaddr_tainted,
 644                     ArmFault::TranslationLL + L1,
 645                     isStage2,
 646                     ArmFault::LpaeTran);
 647             else
 648                 return std::make_shared<DataAbort>(
 649                     currState->vaddr_tainted,
 650                     TlbEntry::DomainType::NoAccess,
 651                     currState->isWrite, ArmFault::TranslationLL + L1,
 652                     isStage2, ArmFault::LpaeTran);
 653         }
 654
 655     }
 656
 657     // Perform lookup (ARM ARM issue C B3.6.6)
 658     if (start_lookup_level == L1) {
 659         n = 5 - tsz;
 660         desc_addr = mbits(ttbr, 39, n) |
 661             (bits(currState->vaddr, n + 26, 30) << 3);
 662         DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
 663                 desc_addr, currState->isSecure ? "s" : "ns");
 664     } else {
 665         // Skip first-level lookup
 666         n = (tsz >= 2 ? 14 - tsz : 12);
 667         desc_addr = mbits(ttbr, 39, n) |
 668             (bits(currState->vaddr, n + 17, 21) << 3);
 669         DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
 670                 desc_addr, currState->isSecure ? "s" : "ns");
 671     }
 672
 673     // Trickbox address check
 674     Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
 675                         currState->vaddr, sizeof(uint64_t), currState->isFetch,
 676                         currState->isWrite, TlbEntry::DomainType::NoAccess,
 677                         start_lookup_level);
 678     if (f) {
 679         DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
 680         if (currState->timing) {
 681             pending = false;
 682             nextWalk(currState->tc);
 683             currState = NULL;
 684         } else {
 685             currState->tc = NULL;
 686             currState->req = NULL;
 687         }
 688         return f;
 689     }
 690
 691     if (currState->sctlr.c == 0) {
 692         flag = Request::UNCACHEABLE;
 693     }
 694
 695     if (currState->isSecure)
 696         flag.set(Request::SECURE);
 697
 698     currState->longDesc.lookupLevel = start_lookup_level;
 699     currState->longDesc.aarch64 = false;
 700     currState->longDesc.grainSize = Grain4KB;
 701
 702     Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
 703                                             : (Event *) &doL2LongDescEvent;
 704
 705     bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
 706                                    sizeof(uint64_t), flag, start_lookup_level,
 707                                    event, &TableWalker::doLongDescriptor);
 708     if (!delayed) {
 709         f = currState->fault;
 710     }
 711
 712     return f;
 713 }
 714
 715 unsigned
 716 TableWalker::adjustTableSizeAArch64(unsigned tsz)
 717 {
 718     if (tsz < 25)
 719         return 25;
 720     if (tsz > 48)
 721         return 48;
 722     return tsz;
 723 }
 724
 725 bool
 726 TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
 727 {
 728     return (currPhysAddrRange != MaxPhysAddrRange &&
 729             bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
 730 }
 731
 732 Fault
 733 TableWalker::processWalkAArch64()
 734 {
 735     assert(currState->aarch64);
 736
 737     DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
 738             currState->vaddr_tainted, currState->tcr);
 739
 740     static const GrainSize GrainMapDefault[] =
 741       { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
 742     static const GrainSize GrainMap_EL1_tg1[] =
 743       { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
 744
 745     statWalkWaitTime.sample(curTick() - currState->startTime);
 746
 747     // Determine TTBR, table size, granule size and phys. address range
 748     Addr ttbr = 0;
 749     int tsz = 0, ps = 0;
 750     GrainSize tg = Grain4KB; // grain size computed from tg* field
 751     bool fault = false;
 752     switch (currState->el) {
 753       case EL0:
 754       case EL1:
 755         switch (bits(currState->vaddr, 63,48)) {
 756           case 0:
 757             DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
 758             ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
 759             tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
 760             tg = GrainMapDefault[currState->tcr.tg0];
 761             if (bits(currState->vaddr, 63, tsz) != 0x0 ||
 762                 currState->tcr.epd0)
 763               fault = true;
 764             break;
 765           case 0xffff:
 766             DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
 767             ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
 768             tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
 769             tg = GrainMap_EL1_tg1[currState->tcr.tg1];
 770             if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
 771                 currState->tcr.epd1)
 772               fault = true;
 773             break;
 774           default:
 775             // top two bytes must be all 0s or all 1s, else invalid addr
 776             fault = true;
 777         }
 778         ps = currState->tcr.ips;
 779         break;
 780       case EL2:
 781       case EL3:
 782         switch(bits(currState->vaddr, 63,48)) {
 783             case 0:
 784                 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
 785                 if (currState->el == EL2)
 786                     ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
 787                 else
 788                     ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
 789                 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
 790                 tg = GrainMapDefault[currState->tcr.tg0];
 791                 break;
 792             default:
 793                 // invalid addr if top two bytes are not all 0s
 794                 fault = true;
 795         }
 796         ps = currState->tcr.ips;
 797         break;
 798     }
 799
 800     if (fault) {
 801         Fault f;
 802         if (currState->isFetch)
 803             f =  std::make_shared<PrefetchAbort>(
 804                 currState->vaddr_tainted,
 805                 ArmFault::TranslationLL + L0, isStage2,
 806                 ArmFault::LpaeTran);
 807         else
 808             f = std::make_shared<DataAbort>(
 809                 currState->vaddr_tainted,
 810                 TlbEntry::DomainType::NoAccess,
 811                 currState->isWrite,
 812                 ArmFault::TranslationLL + L0,
 813                 isStage2, ArmFault::LpaeTran);
 814
 815         if (currState->timing) {
 816             pending = false;
 817             nextWalk(currState->tc);
 818             currState = NULL;
 819         } else {
 820             currState->tc = NULL;
 821             currState->req = NULL;
 822         }
 823         return f;
 824
 825     }
 826
 827     if (tg == ReservedGrain) {
 828         warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
 829                   "DEFINED behavior takes this to mean 4KB granules\n");
 830         tg = Grain4KB;
 831     }
 832
 833     int stride = tg - 3;
 834     LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
 835
 836     // Determine starting lookup level
 837     // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
 838     // in ARM DDI 0487A.  These table values correspond to the cascading tests
 839     // to compute the lookup level and are of the form
 840     // (grain_size + N*stride), for N = {1, 2, 3}.
 841     // A value of 64 will never succeed and a value of 0 will always succeed.
 842     {
 843         struct GrainMap {
 844             GrainSize grain_size;
 845             unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
 846         };
 847         static const GrainMap GM[] = {
 848             { Grain4KB,  { 39, 30,  0, 0 } },
 849             { Grain16KB, { 47, 36, 25, 0 } },
 850             { Grain64KB, { 64, 42, 29, 0 } }
 851         };
 852
 853         const unsigned *lookup = NULL; // points to a lookup_level_cutoff
 854
 855         for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
 856             if (tg == GM[i].grain_size) {
 857                 lookup = GM[i].lookup_level_cutoff;
 858                 break;
 859             }
 860         }
 861         assert(lookup);
 862
 863         for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
 864             if (tsz > lookup[L]) {
 865                 start_lookup_level = (LookupLevel) L;
 866                 break;
 867             }
 868         }
 869         panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
 870                  "Table walker couldn't find lookup level\n");
 871     }
 872
 873     // Determine table base address
 874     int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
 875     Addr base_addr = mbits(ttbr, 47, base_addr_lo);
 876
 877     // Determine physical address size and raise an Address Size Fault if
 878     // necessary
 879     int pa_range = decodePhysAddrRange64(ps);
 880     // Clamp to lower limit
 881     if (pa_range > physAddrRange)
 882         currState->physAddrRange = physAddrRange;
 883     else
 884         currState->physAddrRange = pa_range;
 885     if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
 886         DPRINTF(TLB, "Address size fault before any lookup\n");
 887         Fault f;
 888         if (currState->isFetch)
 889             f = std::make_shared<PrefetchAbort>(
 890                 currState->vaddr_tainted,
 891                 ArmFault::AddressSizeLL + start_lookup_level,
 892                 isStage2,
 893                 ArmFault::LpaeTran);
 894         else
 895             f = std::make_shared<DataAbort>(
 896                 currState->vaddr_tainted,
 897                 TlbEntry::DomainType::NoAccess,
 898                 currState->isWrite,
 899                 ArmFault::AddressSizeLL + start_lookup_level,
 900                 isStage2,
 901                 ArmFault::LpaeTran);
 902
 903
 904         if (currState->timing) {
 905             pending = false;
 906             nextWalk(currState->tc);
 907             currState = NULL;
 908         } else {
 909             currState->tc = NULL;
 910             currState->req = NULL;
 911         }
 912         return f;
 913
 914    }
 915
 916     // Determine descriptor address
 917     Addr desc_addr = base_addr |
 918         (bits(currState->vaddr, tsz - 1,
 919               stride * (3 - start_lookup_level) + tg) << 3);
 920
 921     // Trickbox address check
 922     Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
 923                         currState->vaddr, sizeof(uint64_t), currState->isFetch,
 924                         currState->isWrite, TlbEntry::DomainType::NoAccess,
 925                         start_lookup_level);
 926     if (f) {
 927         DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
 928         if (currState->timing) {
 929             pending = false;
 930             nextWalk(currState->tc);
 931             currState = NULL;
 932         } else {
 933             currState->tc = NULL;
 934             currState->req = NULL;
 935         }
 936         return f;
 937     }
 938
 939     Request::Flags flag = 0;
 940     if (currState->sctlr.c == 0) {
 941         flag = Request::UNCACHEABLE;
 942     }
 943
 944     currState->longDesc.lookupLevel = start_lookup_level;
 945     currState->longDesc.aarch64 = true;
 946     currState->longDesc.grainSize = tg;
 947
 948     if (currState->timing) {
 949         Event *event;
 950         switch (start_lookup_level) {
 951           case L0:
 952             event = (Event *) &doL0LongDescEvent;
 953             break;
 954           case L1:
 955             event = (Event *) &doL1LongDescEvent;
 956             break;
 957           case L2:
 958             event = (Event *) &doL2LongDescEvent;
 959             break;
 960           case L3:
 961             event = (Event *) &doL3LongDescEvent;
 962             break;
 963           default:
 964             panic("Invalid table lookup level");
 965             break;
 966         }
 967         port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
 968                        event, (uint8_t*) &currState->longDesc.data,
 969                        currState->tc->getCpuPtr()->clockPeriod(), flag);
 970         DPRINTF(TLBVerbose,
 971                 "Adding to walker fifo: queue size before adding: %d\n",
 972                 stateQueues[start_lookup_level].size());
 973         stateQueues[start_lookup_level].push_back(currState);
 974         currState = NULL;
 975     } else if (!currState->functional) {
 976         port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
 977                        NULL, (uint8_t*) &currState->longDesc.data,
 978                        currState->tc->getCpuPtr()->clockPeriod(), flag);
 979         doLongDescriptor();
 980         f = currState->fault;
 981     } else {
 982         RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
 983                                      masterId);
 984         PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
 985         pkt->dataStatic((uint8_t*) &currState->longDesc.data);
 986         port->sendFunctional(pkt);
 987         doLongDescriptor();
 988         delete req;
 989         delete pkt;
 990         f = currState->fault;
 991     }
 992
 993     return f;
 994 }
 995
 996 void
 997 TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
 998                       uint8_t texcb, bool s)
 999 {
1000     // Note: tc and sctlr local variables are hiding tc and sctrl class
1001     // variables
1002     DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
1003     te.shareable = false; // default value
1004     te.nonCacheable = false;
1005     te.outerShareable = false;
1006     if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
1007         switch(texcb) {
1008           case 0: // Stongly-ordered
1009             te.nonCacheable = true;
1010             te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1011             te.shareable = true;
1012             te.innerAttrs = 1;
1013             te.outerAttrs = 0;
1014             break;
1015           case 1: // Shareable Device
1016             te.nonCacheable = true;
1017             te.mtype = TlbEntry::MemoryType::Device;
1018             te.shareable = true;
1019             te.innerAttrs = 3;
1020             te.outerAttrs = 0;
1021             break;
1022           case 2: // Outer and Inner Write-Through, no Write-Allocate
1023             te.mtype = TlbEntry::MemoryType::Normal;
1024             te.shareable = s;
1025             te.innerAttrs = 6;
1026             te.outerAttrs = bits(texcb, 1, 0);
1027             break;
1028           case 3: // Outer and Inner Write-Back, no Write-Allocate
1029             te.mtype = TlbEntry::MemoryType::Normal;
1030             te.shareable = s;
1031             te.innerAttrs = 7;
1032             te.outerAttrs = bits(texcb, 1, 0);
1033             break;
1034           case 4: // Outer and Inner Non-cacheable
1035             te.nonCacheable = true;
1036             te.mtype = TlbEntry::MemoryType::Normal;
1037             te.shareable = s;
1038             te.innerAttrs = 0;
1039             te.outerAttrs = bits(texcb, 1, 0);
1040             break;
1041           case 5: // Reserved
1042             panic("Reserved texcb value!\n");
1043             break;
1044           case 6: // Implementation Defined
1045             panic("Implementation-defined texcb value!\n");
1046             break;
1047           case 7: // Outer and Inner Write-Back, Write-Allocate
1048             te.mtype = TlbEntry::MemoryType::Normal;
1049             te.shareable = s;
1050             te.innerAttrs = 5;
1051             te.outerAttrs = 1;
1052             break;
1053           case 8: // Non-shareable Device
1054             te.nonCacheable = true;
1055             te.mtype = TlbEntry::MemoryType::Device;
1056             te.shareable = false;
1057             te.innerAttrs = 3;
1058             te.outerAttrs = 0;
1059             break;
1060           case 9 ... 15:  // Reserved
1061             panic("Reserved texcb value!\n");
1062             break;
1063           case 16 ... 31: // Cacheable Memory
1064             te.mtype = TlbEntry::MemoryType::Normal;
1065             te.shareable = s;
1066             if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1067                 te.nonCacheable = true;
1068             te.innerAttrs = bits(texcb, 1, 0);
1069             te.outerAttrs = bits(texcb, 3, 2);
1070             break;
1071           default:
1072             panic("More than 32 states for 5 bits?\n");
1073         }
1074     } else {
1075         assert(tc);
1076         PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1077                                     currState->tc, !currState->isSecure));
1078         NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1079                                     currState->tc, !currState->isSecure));
1080         DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1081         uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1082         switch(bits(texcb, 2,0)) {
1083           case 0:
1084             curr_tr = prrr.tr0;
1085             curr_ir = nmrr.ir0;
1086             curr_or = nmrr.or0;
1087             te.outerShareable = (prrr.nos0 == 0);
1088             break;
1089           case 1:
1090             curr_tr = prrr.tr1;
1091             curr_ir = nmrr.ir1;
1092             curr_or = nmrr.or1;
1093             te.outerShareable = (prrr.nos1 == 0);
1094             break;
1095           case 2:
1096             curr_tr = prrr.tr2;
1097             curr_ir = nmrr.ir2;
1098             curr_or = nmrr.or2;
1099             te.outerShareable = (prrr.nos2 == 0);
1100             break;
1101           case 3:
1102             curr_tr = prrr.tr3;
1103             curr_ir = nmrr.ir3;
1104             curr_or = nmrr.or3;
1105             te.outerShareable = (prrr.nos3 == 0);
1106             break;
1107           case 4:
1108             curr_tr = prrr.tr4;
1109             curr_ir = nmrr.ir4;
1110             curr_or = nmrr.or4;
1111             te.outerShareable = (prrr.nos4 == 0);
1112             break;
1113           case 5:
1114             curr_tr = prrr.tr5;
1115             curr_ir = nmrr.ir5;
1116             curr_or = nmrr.or5;
1117             te.outerShareable = (prrr.nos5 == 0);
1118             break;
1119           case 6:
1120             panic("Imp defined type\n");
1121           case 7:
1122             curr_tr = prrr.tr7;
1123             curr_ir = nmrr.ir7;
1124             curr_or = nmrr.or7;
1125             te.outerShareable = (prrr.nos7 == 0);
1126             break;
1127         }
1128
1129         switch(curr_tr) {
1130           case 0:
1131             DPRINTF(TLBVerbose, "StronglyOrdered\n");
1132             te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1133             te.nonCacheable = true;
1134             te.innerAttrs = 1;
1135             te.outerAttrs = 0;
1136             te.shareable = true;
1137             break;
1138           case 1:
1139             DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1140                     prrr.ds1, prrr.ds0, s);
1141             te.mtype = TlbEntry::MemoryType::Device;
1142             te.nonCacheable = true;
1143             te.innerAttrs = 3;
1144             te.outerAttrs = 0;
1145             if (prrr.ds1 && s)
1146                 te.shareable = true;
1147             if (prrr.ds0 && !s)
1148                 te.shareable = true;
1149             break;
1150           case 2:
1151             DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1152                     prrr.ns1, prrr.ns0, s);
1153             te.mtype = TlbEntry::MemoryType::Normal;
1154             if (prrr.ns1 && s)
1155                 te.shareable = true;
1156             if (prrr.ns0 && !s)
1157                 te.shareable = true;
1158             break;
1159           case 3:
1160             panic("Reserved type");
1161         }
1162
1163         if (te.mtype == TlbEntry::MemoryType::Normal){
1164             switch(curr_ir) {
1165               case 0:
1166                 te.nonCacheable = true;
1167                 te.innerAttrs = 0;
1168                 break;
1169               case 1:
1170                 te.innerAttrs = 5;
1171                 break;
1172               case 2:
1173                 te.innerAttrs = 6;
1174                 break;
1175               case 3:
1176                 te.innerAttrs = 7;
1177                 break;
1178             }
1179
1180             switch(curr_or) {
1181               case 0:
1182                 te.nonCacheable = true;
1183                 te.outerAttrs = 0;
1184                 break;
1185               case 1:
1186                 te.outerAttrs = 1;
1187                 break;
1188               case 2:
1189                 te.outerAttrs = 2;
1190                 break;
1191               case 3:
1192                 te.outerAttrs = 3;
1193                 break;
1194             }
1195         }
1196     }
1197     DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1198             "outerAttrs: %d\n",
1199             te.shareable, te.innerAttrs, te.outerAttrs);
1200     te.setAttributes(false);
1201 }
1202
1203 void
1204 TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1205     LongDescriptor &lDescriptor)
1206 {
1207     assert(_haveLPAE);
1208
1209     uint8_t attr;
1210     uint8_t sh = lDescriptor.sh();
1211     // Different format and source of attributes if this is a stage 2
1212     // translation
1213     if (isStage2) {
1214         attr = lDescriptor.memAttr();
1215         uint8_t attr_3_2 = (attr >> 2) & 0x3;
1216         uint8_t attr_1_0 =  attr       & 0x3;
1217
1218         DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1219
1220         if (attr_3_2 == 0) {
1221             te.mtype        = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1222                                             : TlbEntry::MemoryType::Device;
1223             te.outerAttrs   = 0;
1224             te.innerAttrs   = attr_1_0 == 0 ? 1 : 3;
1225             te.nonCacheable = true;
1226         } else {
1227             te.mtype        = TlbEntry::MemoryType::Normal;
1228             te.outerAttrs   = attr_3_2 == 1 ? 0 :
1229                               attr_3_2 == 2 ? 2 : 1;
1230             te.innerAttrs   = attr_1_0 == 1 ? 0 :
1231                               attr_1_0 == 2 ? 6 : 5;
1232             te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1233         }
1234     } else {
1235         uint8_t attrIndx = lDescriptor.attrIndx();
1236
1237         // LPAE always uses remapping of memory attributes, irrespective of the
1238         // value of SCTLR.TRE
1239         MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1240         int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1241                                                 !currState->isSecure);
1242         uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1243         attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1244         uint8_t attr_7_4 = bits(attr, 7, 4);
1245         uint8_t attr_3_0 = bits(attr, 3, 0);
1246         DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1247
1248         // Note: the memory subsystem only cares about the 'cacheable' memory
1249         // attribute. The other attributes are only used to fill the PAR register
1250         // accordingly to provide the illusion of full support
1251         te.nonCacheable = false;
1252
1253         switch (attr_7_4) {
1254           case 0x0:
1255             // Strongly-ordered or Device memory
1256             if (attr_3_0 == 0x0)
1257                 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1258             else if (attr_3_0 == 0x4)
1259                 te.mtype = TlbEntry::MemoryType::Device;
1260             else
1261                 panic("Unpredictable behavior\n");
1262             te.nonCacheable = true;
1263             te.outerAttrs   = 0;
1264             break;
1265           case 0x4:
1266             // Normal memory, Outer Non-cacheable
1267             te.mtype = TlbEntry::MemoryType::Normal;
1268             te.outerAttrs = 0;
1269             if (attr_3_0 == 0x4)
1270                 // Inner Non-cacheable
1271                 te.nonCacheable = true;
1272             else if (attr_3_0 < 0x8)
1273                 panic("Unpredictable behavior\n");
1274             break;
1275           case 0x8:
1276           case 0x9:
1277           case 0xa:
1278           case 0xb:
1279           case 0xc:
1280           case 0xd:
1281           case 0xe:
1282           case 0xf:
1283             if (attr_7_4 & 0x4) {
1284                 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1285             } else {
1286                 te.outerAttrs = 0x2;
1287             }
1288             // Normal memory, Outer Cacheable
1289             te.mtype = TlbEntry::MemoryType::Normal;
1290             if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1291                 panic("Unpredictable behavior\n");
1292             break;
1293           default:
1294             panic("Unpredictable behavior\n");
1295             break;
1296         }
1297
1298         switch (attr_3_0) {
1299           case 0x0:
1300             te.innerAttrs = 0x1;
1301             break;
1302           case 0x4:
1303             te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1304             break;
1305           case 0x8:
1306           case 0x9:
1307           case 0xA:
1308           case 0xB:
1309             te.innerAttrs = 6;
1310             break;
1311           case 0xC:
1312           case 0xD:
1313           case 0xE:
1314           case 0xF:
1315             te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1316             break;
1317           default:
1318             panic("Unpredictable behavior\n");
1319             break;
1320         }
1321     }
1322
1323     te.outerShareable = sh == 2;
1324     te.shareable       = (sh & 0x2) ? true : false;
1325     te.setAttributes(true);
1326     te.attributes |= (uint64_t) attr << 56;
1327 }
1328
1329 void
1330 TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1331                              uint8_t sh)
1332 {
1333     DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1334
1335     // Select MAIR
1336     uint64_t mair;
1337     switch (currState->el) {
1338       case EL0:
1339       case EL1:
1340         mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1341         break;
1342       case EL2:
1343         mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1344         break;
1345       case EL3:
1346         mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1347         break;
1348       default:
1349         panic("Invalid exception level");
1350         break;
1351     }
1352
1353     // Select attributes
1354     uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1355     uint8_t attr_lo = bits(attr, 3, 0);
1356     uint8_t attr_hi = bits(attr, 7, 4);
1357
1358     // Memory type
1359     te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1360
1361     // Cacheability
1362     te.nonCacheable = false;
1363     if (te.mtype == TlbEntry::MemoryType::Device ||  // Device memory
1364         attr_hi == 0x8 ||  // Normal memory, Outer Non-cacheable
1365         attr_lo == 0x8) {  // Normal memory, Inner Non-cacheable
1366         te.nonCacheable = true;
1367     }
1368
1369     te.shareable       = sh == 2;
1370     te.outerShareable = (sh & 0x2) ? true : false;
1371     // Attributes formatted according to the 64-bit PAR
1372     te.attributes = ((uint64_t) attr << 56) |
1373         (1 << 11) |     // LPAE bit
1374         (te.ns << 9) |  // NS bit
1375         (sh << 7);
1376 }
1377
1378 void
1379 TableWalker::doL1Descriptor()
1380 {
1381     if (currState->fault != NoFault) {
1382         return;
1383     }
1384
1385     DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1386             currState->vaddr_tainted, currState->l1Desc.data);
1387     TlbEntry te;
1388
1389     switch (currState->l1Desc.type()) {
1390       case L1Descriptor::Ignore:
1391       case L1Descriptor::Reserved:
1392         if (!currState->timing) {
1393             currState->tc = NULL;
1394             currState->req = NULL;
1395         }
1396         DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1397         if (currState->isFetch)
1398             currState->fault =
1399                 std::make_shared<PrefetchAbort>(
1400                     currState->vaddr_tainted,
1401                     ArmFault::TranslationLL + L1,
1402                     isStage2,
1403                     ArmFault::VmsaTran);
1404         else
1405             currState->fault =
1406                 std::make_shared<DataAbort>(
1407                     currState->vaddr_tainted,
1408                     TlbEntry::DomainType::NoAccess,
1409                     currState->isWrite,
1410                     ArmFault::TranslationLL + L1, isStage2,
1411                     ArmFault::VmsaTran);
1412         return;
1413       case L1Descriptor::Section:
1414         if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1415             /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1416               * enabled if set, do l1.Desc.setAp0() instead of generating
1417               * AccessFlag0
1418               */
1419
1420             currState->fault = std::make_shared<DataAbort>(
1421                 currState->vaddr_tainted,
1422                 currState->l1Desc.domain(),
1423                 currState->isWrite,
1424                 ArmFault::AccessFlagLL + L1,
1425                 isStage2,
1426                 ArmFault::VmsaTran);
1427         }
1428         if (currState->l1Desc.supersection()) {
1429             panic("Haven't implemented supersections\n");
1430         }
1431         insertTableEntry(currState->l1Desc, false);
1432         return;
1433       case L1Descriptor::PageTable:
1434         {
1435             Addr l2desc_addr;
1436             l2desc_addr = currState->l1Desc.l2Addr() |
1437                 (bits(currState->vaddr, 19, 12) << 2);
1438             DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1439                     l2desc_addr, currState->isSecure ? "s" : "ns");
1440
1441             // Trickbox address check
1442             currState->fault = tlb->walkTrickBoxCheck(
1443                 l2desc_addr, currState->isSecure, currState->vaddr,
1444                 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1445                 currState->l1Desc.domain(), L2);
1446
1447             if (currState->fault) {
1448                 if (!currState->timing) {
1449                     currState->tc = NULL;
1450                     currState->req = NULL;
1451                 }
1452                 return;
1453             }
1454
1455             Request::Flags flag = 0;
1456             if (currState->isSecure)
1457                 flag.set(Request::SECURE);
1458
1459             bool delayed;
1460             delayed = fetchDescriptor(l2desc_addr,
1461                                       (uint8_t*)&currState->l2Desc.data,
1462                                       sizeof(uint32_t), flag, -1, &doL2DescEvent,
1463                                       &TableWalker::doL2Descriptor);
1464             if (delayed) {
1465                 currState->delayed = true;
1466             }
1467
1468             return;
1469         }
1470       default:
1471         panic("A new type in a 2 bit field?\n");
1472     }
1473 }
1474
1475 void
1476 TableWalker::doLongDescriptor()
1477 {
1478     if (currState->fault != NoFault) {
1479         return;
1480     }
1481
1482     DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1483             currState->longDesc.lookupLevel, currState->vaddr_tainted,
1484             currState->longDesc.data,
1485             currState->aarch64 ? "AArch64" : "long-desc.");
1486
1487     if ((currState->longDesc.type() == LongDescriptor::Block) ||
1488         (currState->longDesc.type() == LongDescriptor::Page)) {
1489         DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1490                 "xn: %d, ap: %d, af: %d, type: %d\n",
1491                 currState->longDesc.lookupLevel,
1492                 currState->longDesc.data,
1493                 currState->longDesc.pxn(),
1494                 currState->longDesc.xn(),
1495                 currState->longDesc.ap(),
1496                 currState->longDesc.af(),
1497                 currState->longDesc.type());
1498     } else {
1499         DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1500                 currState->longDesc.lookupLevel,
1501                 currState->longDesc.data,
1502                 currState->longDesc.type());
1503     }
1504
1505     TlbEntry te;
1506
1507     switch (currState->longDesc.type()) {
1508       case LongDescriptor::Invalid:
1509         if (!currState->timing) {
1510             currState->tc = NULL;
1511             currState->req = NULL;
1512         }
1513
1514         DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1515                 currState->longDesc.lookupLevel,
1516                 ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1517         if (currState->isFetch)
1518             currState->fault = std::make_shared<PrefetchAbort>(
1519                 currState->vaddr_tainted,
1520                 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1521                 isStage2,
1522                 ArmFault::LpaeTran);
1523         else
1524             currState->fault = std::make_shared<DataAbort>(
1525                 currState->vaddr_tainted,
1526                 TlbEntry::DomainType::NoAccess,
1527                 currState->isWrite,
1528                 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1529                 isStage2,
1530                 ArmFault::LpaeTran);
1531         return;
1532       case LongDescriptor::Block:
1533       case LongDescriptor::Page:
1534         {
1535             bool fault = false;
1536             bool aff = false;
1537             // Check for address size fault
1538             if (checkAddrSizeFaultAArch64(
1539                     mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1540                           currState->longDesc.offsetBits()),
1541                     currState->physAddrRange)) {
1542                 fault = true;
1543                 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1544                         currState->longDesc.lookupLevel);
1545             // Check for access fault
1546             } else if (currState->longDesc.af() == 0) {
1547                 fault = true;
1548                 DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1549                         currState->longDesc.lookupLevel);
1550                 aff = true;
1551             }
1552             if (fault) {
1553                 if (currState->isFetch)
1554                     currState->fault = std::make_shared<PrefetchAbort>(
1555                         currState->vaddr_tainted,
1556                         (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1557                         currState->longDesc.lookupLevel,
1558                         isStage2,
1559                         ArmFault::LpaeTran);
1560                 else
1561                     currState->fault = std::make_shared<DataAbort>(
1562                         currState->vaddr_tainted,
1563                         TlbEntry::DomainType::NoAccess, currState->isWrite,
1564                         (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1565                         currState->longDesc.lookupLevel,
1566                         isStage2,
1567                         ArmFault::LpaeTran);
1568             } else {
1569                 insertTableEntry(currState->longDesc, true);
1570             }
1571         }
1572         return;
1573       case LongDescriptor::Table:
1574         {
1575             // Set hierarchical permission flags
1576             currState->secureLookup = currState->secureLookup &&
1577                 currState->longDesc.secureTable();
1578             currState->rwTable = currState->rwTable &&
1579                 currState->longDesc.rwTable();
1580             currState->userTable = currState->userTable &&
1581                 currState->longDesc.userTable();
1582             currState->xnTable = currState->xnTable ||
1583                 currState->longDesc.xnTable();
1584             currState->pxnTable = currState->pxnTable ||
1585                 currState->longDesc.pxnTable();
1586
1587             // Set up next level lookup
1588             Addr next_desc_addr = currState->longDesc.nextDescAddr(
1589                 currState->vaddr);
1590
1591             DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1592                     currState->longDesc.lookupLevel,
1593                     currState->longDesc.lookupLevel + 1,
1594                     next_desc_addr,
1595                     currState->secureLookup ? "s" : "ns");
1596
1597             // Check for address size fault
1598             if (currState->aarch64 && checkAddrSizeFaultAArch64(
1599                     next_desc_addr, currState->physAddrRange)) {
1600                 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1601                         currState->longDesc.lookupLevel);
1602                 if (currState->isFetch)
1603                     currState->fault = std::make_shared<PrefetchAbort>(
1604                         currState->vaddr_tainted,
1605                         ArmFault::AddressSizeLL
1606                         + currState->longDesc.lookupLevel,
1607                         isStage2,
1608                         ArmFault::LpaeTran);
1609                 else
1610                     currState->fault = std::make_shared<DataAbort>(
1611                         currState->vaddr_tainted,
1612                         TlbEntry::DomainType::NoAccess, currState->isWrite,
1613                         ArmFault::AddressSizeLL
1614                         + currState->longDesc.lookupLevel,
1615                         isStage2,
1616                         ArmFault::LpaeTran);
1617                 return;
1618             }
1619
1620             // Trickbox address check
1621             currState->fault = tlb->walkTrickBoxCheck(
1622                             next_desc_addr, currState->vaddr,
1623                             currState->vaddr, sizeof(uint64_t),
1624                             currState->isFetch, currState->isWrite,
1625                             TlbEntry::DomainType::Client,
1626                             toLookupLevel(currState->longDesc.lookupLevel +1));
1627
1628             if (currState->fault) {
1629                 if (!currState->timing) {
1630                     currState->tc = NULL;
1631                     currState->req = NULL;
1632                 }
1633                 return;
1634             }
1635
1636             Request::Flags flag = 0;
1637             if (currState->secureLookup)
1638                 flag.set(Request::SECURE);
1639
1640             currState->longDesc.lookupLevel =
1641                 (LookupLevel) (currState->longDesc.lookupLevel + 1);
1642             Event *event = NULL;
1643             switch (currState->longDesc.lookupLevel) {
1644               case L1:
1645                 assert(currState->aarch64);
1646                 event = &doL1LongDescEvent;
1647                 break;
1648               case L2:
1649                 event = &doL2LongDescEvent;
1650                 break;
1651               case L3:
1652                 event = &doL3LongDescEvent;
1653                 break;
1654               default:
1655                 panic("Wrong lookup level in table walk\n");
1656                 break;
1657             }
1658
1659             bool delayed;
1660             delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1661                                       sizeof(uint64_t), flag, -1, event,
1662                                       &TableWalker::doLongDescriptor);
1663             if (delayed) {
1664                  currState->delayed = true;
1665             }
1666         }
1667         return;
1668       default:
1669         panic("A new type in a 2 bit field?\n");
1670     }
1671 }
1672
1673 void
1674 TableWalker::doL2Descriptor()
1675 {
1676     if (currState->fault != NoFault) {
1677         return;
1678     }
1679
1680     DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1681             currState->vaddr_tainted, currState->l2Desc.data);
1682     TlbEntry te;
1683
1684     if (currState->l2Desc.invalid()) {
1685         DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1686         if (!currState->timing) {
1687             currState->tc = NULL;
1688             currState->req = NULL;
1689         }
1690         if (currState->isFetch)
1691             currState->fault = std::make_shared<PrefetchAbort>(
1692                     currState->vaddr_tainted,
1693                     ArmFault::TranslationLL + L2,
1694                     isStage2,
1695                     ArmFault::VmsaTran);
1696         else
1697             currState->fault = std::make_shared<DataAbort>(
1698                 currState->vaddr_tainted, currState->l1Desc.domain(),
1699                 currState->isWrite, ArmFault::TranslationLL + L2,
1700                 isStage2,
1701                 ArmFault::VmsaTran);
1702         return;
1703     }
1704
1705     if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1706         /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1707           * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1708           */
1709          DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1710                  currState->sctlr.afe, currState->l2Desc.ap());
1711
1712         currState->fault = std::make_shared<DataAbort>(
1713             currState->vaddr_tainted,
1714             TlbEntry::DomainType::NoAccess, currState->isWrite,
1715             ArmFault::AccessFlagLL + L2, isStage2,
1716             ArmFault::VmsaTran);
1717     }
1718
1719     insertTableEntry(currState->l2Desc, false);
1720 }
1721
1722 void
1723 TableWalker::doL1DescriptorWrapper()
1724 {
1725     currState = stateQueues[L1].front();
1726     currState->delayed = false;
1727     // if there's a stage2 translation object we don't need it any more
1728     if (currState->stage2Tran) {
1729         delete currState->stage2Tran;
1730         currState->stage2Tran = NULL;
1731     }
1732
1733
1734     DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1735     DPRINTF(TLBVerbose, "L1 Desc object      data: %08x\n",currState->l1Desc.data);
1736
1737     DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1738     doL1Descriptor();
1739
1740     stateQueues[L1].pop_front();
1741     // Check if fault was generated
1742     if (currState->fault != NoFault) {
1743         currState->transState->finish(currState->fault, currState->req,
1744                                       currState->tc, currState->mode);
1745         statWalksShortTerminatedAtLevel[0]++;
1746
1747         pending = false;
1748         nextWalk(currState->tc);
1749
1750         currState->req = NULL;
1751         currState->tc = NULL;
1752         currState->delayed = false;
1753         delete currState;
1754     }
1755     else if (!currState->delayed) {
1756         // delay is not set so there is no L2 to do
1757         // Don't finish the translation if a stage 2 look up is underway
1758         if (!currState->doingStage2) {
1759             statWalkServiceTime.sample(curTick() - currState->startTime);
1760             DPRINTF(TLBVerbose, "calling translateTiming again\n");
1761             currState->fault = tlb->translateTiming(currState->req, currState->tc,
1762                 currState->transState, currState->mode);
1763             statWalksShortTerminatedAtLevel[0]++;
1764         }
1765
1766         pending = false;
1767         nextWalk(currState->tc);
1768
1769         currState->req = NULL;
1770         currState->tc = NULL;
1771         currState->delayed = false;
1772         delete currState;
1773     } else {
1774         // need to do L2 descriptor
1775         stateQueues[L2].push_back(currState);
1776     }
1777     currState = NULL;
1778 }
1779
1780 void
1781 TableWalker::doL2DescriptorWrapper()
1782 {
1783     currState = stateQueues[L2].front();
1784     assert(currState->delayed);
1785     // if there's a stage2 translation object we don't need it any more
1786     if (currState->stage2Tran) {
1787         delete currState->stage2Tran;
1788         currState->stage2Tran = NULL;
1789     }
1790
1791     DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1792             currState->vaddr_tainted);
1793     doL2Descriptor();
1794
1795     // Check if fault was generated
1796     if (currState->fault != NoFault) {
1797         currState->transState->finish(currState->fault, currState->req,
1798                                       currState->tc, currState->mode);
1799         statWalksShortTerminatedAtLevel[1]++;
1800     }
1801     else {
1802         // Don't finish the translation if a stage 2 look up is underway
1803         if (!currState->doingStage2) {
1804             statWalkServiceTime.sample(curTick() - currState->startTime);
1805             DPRINTF(TLBVerbose, "calling translateTiming again\n");
1806             currState->fault = tlb->translateTiming(currState->req,
1807                 currState->tc, currState->transState, currState->mode);
1808             statWalksShortTerminatedAtLevel[1]++;
1809         }
1810     }
1811
1812
1813     stateQueues[L2].pop_front();
1814     pending = false;
1815     nextWalk(currState->tc);
1816
1817     currState->req = NULL;
1818     currState->tc = NULL;
1819     currState->delayed = false;
1820
1821     delete currState;
1822     currState = NULL;
1823 }
1824
1825 void
1826 TableWalker::doL0LongDescriptorWrapper()
1827 {
1828     doLongDescriptorWrapper(L0);
1829 }
1830
1831 void
1832 TableWalker::doL1LongDescriptorWrapper()
1833 {
1834     doLongDescriptorWrapper(L1);
1835 }
1836
1837 void
1838 TableWalker::doL2LongDescriptorWrapper()
1839 {
1840     doLongDescriptorWrapper(L2);
1841 }
1842
1843 void
1844 TableWalker::doL3LongDescriptorWrapper()
1845 {
1846     doLongDescriptorWrapper(L3);
1847 }
1848
1849 void
1850 TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1851 {
1852     currState = stateQueues[curr_lookup_level].front();
1853     assert(curr_lookup_level == currState->longDesc.lookupLevel);
1854     currState->delayed = false;
1855
1856     // if there's a stage2 translation object we don't need it any more
1857     if (currState->stage2Tran) {
1858         delete currState->stage2Tran;
1859         currState->stage2Tran = NULL;
1860     }
1861
1862     DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1863             currState->vaddr_tainted);
1864     doLongDescriptor();
1865
1866     stateQueues[curr_lookup_level].pop_front();
1867
1868     if (currState->fault != NoFault) {
1869         // A fault was generated
1870         currState->transState->finish(currState->fault, currState->req,
1871                                       currState->tc, currState->mode);
1872
1873         pending = false;
1874         nextWalk(currState->tc);
1875
1876         currState->req = NULL;
1877         currState->tc = NULL;
1878         currState->delayed = false;
1879         delete currState;
1880     } else if (!currState->delayed) {
1881         // No additional lookups required
1882         // Don't finish the translation if a stage 2 look up is underway
1883         if (!currState->doingStage2) {
1884             DPRINTF(TLBVerbose, "calling translateTiming again\n");
1885             statWalkServiceTime.sample(curTick() - currState->startTime);
1886             currState->fault = tlb->translateTiming(currState->req, currState->tc,
1887                                                     currState->transState,
1888                                                     currState->mode);
1889             statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1890         }
1891
1892         pending = false;
1893         nextWalk(currState->tc);
1894
1895         currState->req = NULL;
1896         currState->tc = NULL;
1897         currState->delayed = false;
1898         delete currState;
1899     } else {
1900         if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1901             panic("Max. number of lookups already reached in table walk\n");
1902         // Need to perform additional lookups
1903         stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1904     }
1905     currState = NULL;
1906 }
1907
1908
1909 void
1910 TableWalker::nextWalk(ThreadContext *tc)
1911 {
1912     if (pendingQueue.size())
1913         schedule(doProcessEvent, clockEdge(Cycles(1)));
1914     else
1915         completeDrain();
1916 }
1917
1918 bool
1919 TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1920     Request::Flags flags, int queueIndex, Event *event,
1921     void (TableWalker::*doDescriptor)())
1922 {
1923     bool isTiming = currState->timing;
1924
1925     // do the requests for the page table descriptors have to go through the
1926     // second stage MMU
1927     if (currState->stage2Req) {
1928         Fault fault;
1929         flags = flags | TLB::MustBeOne;
1930
1931         if (isTiming) {
1932             Stage2MMU::Stage2Translation *tran = new
1933                 Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1934                                              currState->vaddr);
1935             currState->stage2Tran = tran;
1936             stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
1937                                      flags);
1938             fault = tran->fault;
1939         } else {
1940             fault = stage2Mmu->readDataUntimed(currState->tc,
1941                 currState->vaddr, descAddr, data, numBytes, flags,
1942                 currState->functional);
1943         }
1944
1945         if (fault != NoFault) {
1946             currState->fault = fault;
1947         }
1948         if (isTiming) {
1949             if (queueIndex >= 0) {
1950                 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1951                         stateQueues[queueIndex].size());
1952                 stateQueues[queueIndex].push_back(currState);
1953                 currState = NULL;
1954             }
1955         } else {
1956             (this->*doDescriptor)();
1957         }
1958     } else {
1959         if (isTiming) {
1960             port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
1961                            currState->tc->getCpuPtr()->clockPeriod(),flags);
1962             if (queueIndex >= 0) {
1963                 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1964                         stateQueues[queueIndex].size());
1965                 stateQueues[queueIndex].push_back(currState);
1966                 currState = NULL;
1967             }
1968         } else if (!currState->functional) {
1969             port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
1970                            currState->tc->getCpuPtr()->clockPeriod(), flags);
1971             (this->*doDescriptor)();
1972         } else {
1973             RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1974             req->taskId(ContextSwitchTaskId::DMA);
1975             PacketPtr  pkt = new Packet(req, MemCmd::ReadReq);
1976             pkt->dataStatic(data);
1977             port->sendFunctional(pkt);
1978             (this->*doDescriptor)();
1979             delete req;
1980             delete pkt;
1981         }
1982     }
1983     return (isTiming);
1984 }
1985
1986 void
1987 TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1988 {
1989     TlbEntry te;
1990
1991     // Create and fill a new page table entry
1992     te.valid          = true;
1993     te.longDescFormat = longDescriptor;
1994     te.isHyp          = currState->isHyp;
1995     te.asid           = currState->asid;
1996     te.vmid           = currState->vmid;
1997     te.N              = descriptor.offsetBits();
1998     te.vpn            = currState->vaddr >> te.N;
1999     te.size           = (1<<te.N) - 1;
2000     te.pfn            = descriptor.pfn();
2001     te.domain         = descriptor.domain();
2002     te.lookupLevel    = descriptor.lookupLevel;
2003     te.ns             = !descriptor.secure(haveSecurity, currState) || isStage2;
2004     te.nstid          = !currState->isSecure;
2005     te.xn             = descriptor.xn();
2006     if (currState->aarch64)
2007         te.el         = currState->el;
2008     else
2009         te.el         = 1;
2010
2011     statPageSizes[pageSizeNtoStatBin(te.N)]++;
2012     statRequestOrigin[COMPLETED][currState->isFetch]++;
2013
2014     // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
2015     // as global
2016     te.global         = descriptor.global(currState) || isStage2;
2017     if (longDescriptor) {
2018         LongDescriptor lDescriptor =
2019             dynamic_cast<LongDescriptor &>(descriptor);
2020
2021         te.xn |= currState->xnTable;
2022         te.pxn = currState->pxnTable || lDescriptor.pxn();
2023         if (isStage2) {
2024             // this is actually the HAP field, but its stored in the same bit
2025             // possitions as the AP field in a stage 1 translation.
2026             te.hap = lDescriptor.ap();
2027         } else {
2028            te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2029                (currState->userTable && (descriptor.ap() & 0x1));
2030         }
2031         if (currState->aarch64)
2032             memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2033                             currState->longDesc.sh());
2034         else
2035             memAttrsLPAE(currState->tc, te, lDescriptor);
2036     } else {
2037         te.ap = descriptor.ap();
2038         memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2039                  descriptor.shareable());
2040     }
2041
2042     // Debug output
2043     DPRINTF(TLB, descriptor.dbgHeader().c_str());
2044     DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2045             te.N, te.pfn, te.size, te.global, te.valid);
2046     DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2047             "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2048             te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2049             te.nonCacheable, te.ns);
2050     DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2051             descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2052             descriptor.getRawData());
2053
2054     // Insert the entry into the TLB
2055     tlb->insert(currState->vaddr, te);
2056     if (!currState->timing) {
2057         currState->tc  = NULL;
2058         currState->req = NULL;
2059     }
2060 }
2061
2062 ArmISA::TableWalker *
2063 ArmTableWalkerParams::create()
2064 {
2065     return new ArmISA::TableWalker(this);
2066 }
2067
2068 LookupLevel
2069 TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2070 {
2071     switch (lookup_level_as_int) {
2072       case L1:
2073         return L1;
2074       case L2:
2075         return L2;
2076       case L3:
2077         return L3;
2078       default:
2079         panic("Invalid lookup level conversion");
2080     }
2081 }
2082
2083 /* this method keeps track of the table walker queue's residency, so
2084  * needs to be called whenever requests start and complete. */
2085 void
2086 TableWalker::pendingChange()
2087 {
2088     unsigned n = pendingQueue.size();
2089     if ((currState != NULL) && (currState != pendingQueue.front())) {
2090         ++n;
2091     }
2092
2093     if (n != pendingReqs) {
2094         Tick now = curTick();
2095         statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2096         pendingReqs = n;
2097         pendingChangeTick = now;
2098     }
2099 }
2100
2101 uint8_t
2102 TableWalker::pageSizeNtoStatBin(uint8_t N)
2103 {
2104     /* for statPageSizes */
2105     switch(N) {
2106         case 12: return 0; // 4K
2107         case 14: return 1; // 16K (using 16K granule in v8-64)
2108         case 16: return 2; // 64K
2109         case 20: return 3; // 1M
2110         case 21: return 4; // 2M-LPAE
2111         case 24: return 5; // 16M
2112         case 25: return 6; // 32M (using 16K granule in v8-64)
2113         case 29: return 7; // 512M (using 64K granule in v8-64)
2114         case 30: return 8; // 1G-LPAE
2115         default:
2116             panic("unknown page size");
2117             return 255;
2118     }
2119 }
2120
2121 void
2122 TableWalker::regStats()
2123 {
2124     statWalks
2125         .name(name() + ".walks")
2126         .desc("Table walker walks requested")
2127         ;
2128
2129     statWalksShortDescriptor
2130         .name(name() + ".walksShort")
2131         .desc("Table walker walks initiated with short descriptors")
2132         .flags(Stats::nozero)
2133         ;
2134
2135     statWalksLongDescriptor
2136         .name(name() + ".walksLong")
2137         .desc("Table walker walks initiated with long descriptors")
2138         .flags(Stats::nozero)
2139         ;
2140
2141     statWalksShortTerminatedAtLevel
2142         .init(2)
2143         .name(name() + ".walksShortTerminationLevel")
2144         .desc("Level at which table walker walks "
2145               "with short descriptors terminate")
2146         .flags(Stats::nozero)
2147         ;
2148     statWalksShortTerminatedAtLevel.subname(0, "Level1");
2149     statWalksShortTerminatedAtLevel.subname(1, "Level2");
2150
2151     statWalksLongTerminatedAtLevel
2152         .init(4)
2153         .name(name() + ".walksLongTerminationLevel")
2154         .desc("Level at which table walker walks "
2155               "with long descriptors terminate")
2156         .flags(Stats::nozero)
2157         ;
2158     statWalksLongTerminatedAtLevel.subname(0, "Level0");
2159     statWalksLongTerminatedAtLevel.subname(1, "Level1");
2160     statWalksLongTerminatedAtLevel.subname(2, "Level2");
2161     statWalksLongTerminatedAtLevel.subname(3, "Level3");
2162
2163     statSquashedBefore
2164         .name(name() + ".walksSquashedBefore")
2165         .desc("Table walks squashed before starting")
2166         .flags(Stats::nozero)
2167         ;
2168
2169     statSquashedAfter
2170         .name(name() + ".walksSquashedAfter")
2171         .desc("Table walks squashed after completion")
2172         .flags(Stats::nozero)
2173         ;
2174
2175     statWalkWaitTime
2176         .init(16)
2177         .name(name() + ".walkWaitTime")
2178         .desc("Table walker wait (enqueue to first request) latency")
2179         .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2180         ;
2181
2182     statWalkServiceTime
2183         .init(16)
2184         .name(name() + ".walkCompletionTime")
2185         .desc("Table walker service (enqueue to completion) latency")
2186         .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2187         ;
2188
2189     statPendingWalks
2190         .init(16)
2191         .name(name() + ".walksPending")
2192         .desc("Table walker pending requests distribution")
2193         .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2194         ;
2195
2196     statPageSizes // see DDI 0487A D4-1661
2197         .init(9)
2198         .name(name() + ".walkPageSizes")
2199         .desc("Table walker page sizes translated")
2200         .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2201         ;
2202     statPageSizes.subname(0, "4K");
2203     statPageSizes.subname(1, "16K");
2204     statPageSizes.subname(2, "64K");
2205     statPageSizes.subname(3, "1M");
2206     statPageSizes.subname(4, "2M");
2207     statPageSizes.subname(5, "16M");
2208     statPageSizes.subname(6, "32M");
2209     statPageSizes.subname(7, "512M");
2210     statPageSizes.subname(8, "1G");
2211
2212     statRequestOrigin
2213         .init(2,2) // Instruction/Data, requests/completed
2214         .name(name() + ".walkRequestOrigin")
2215         .desc("Table walker requests started/completed, data/inst")
2216         .flags(Stats::total)
2217         ;
2218     statRequestOrigin.subname(0,"Requested");
2219     statRequestOrigin.subname(1,"Completed");
2220     statRequestOrigin.ysubname(0,"Data");
2221     statRequestOrigin.ysubname(1,"Inst");
2222 }