src/arch/x86/tlb.cc

   1 /*
   2  * Copyright (c) 2007-2008 The Hewlett-Packard Development Company
   3  * All rights reserved.
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include "arch/x86/tlb.hh"
  39
  40 #include <cstring>
  41 #include <memory>
  42
  43 #include "arch/x86/faults.hh"
  44 #include "arch/x86/insts/microldstop.hh"
  45 #include "arch/x86/pagetable_walker.hh"
  46 #include "arch/x86/pseudo_inst_abi.hh"
  47 #include "arch/x86/regs/misc.hh"
  48 #include "arch/x86/regs/msr.hh"
  49 #include "arch/x86/x86_traits.hh"
  50 #include "base/trace.hh"
  51 #include "cpu/thread_context.hh"
  52 #include "debug/TLB.hh"
  53 #include "mem/packet_access.hh"
  54 #include "mem/page_table.hh"
  55 #include "mem/request.hh"
  56 #include "sim/full_system.hh"
  57 #include "sim/process.hh"
  58 #include "sim/pseudo_inst.hh"
  59
  60 namespace X86ISA {
  61
  62 TLB::TLB(const Params *p)
  63     : BaseTLB(p), configAddress(0), size(p->size),
  64       tlb(size), lruSeq(0), m5opRange(p->system->m5opRange())
  65 {
  66     if (!size)
  67         fatal("TLBs must have a non-zero size.\n");
  68
  69     for (int x = 0; x < size; x++) {
  70         tlb[x].trieHandle = NULL;
  71         freeList.push_back(&tlb[x]);
  72     }
  73
  74     walker = p->walker;
  75     walker->setTLB(this);
  76 }
  77
  78 void
  79 TLB::evictLRU()
  80 {
  81     // Find the entry with the lowest (and hence least recently updated)
  82     // sequence number.
  83
  84     unsigned lru = 0;
  85     for (unsigned i = 1; i < size; i++) {
  86         if (tlb[i].lruSeq < tlb[lru].lruSeq)
  87             lru = i;
  88     }
  89
  90     assert(tlb[lru].trieHandle);
  91     trie.remove(tlb[lru].trieHandle);
  92     tlb[lru].trieHandle = NULL;
  93     freeList.push_back(&tlb[lru]);
  94 }
  95
  96 TlbEntry *
  97 TLB::insert(Addr vpn, const TlbEntry &entry)
  98 {
  99     // If somebody beat us to it, just use that existing entry.
 100     TlbEntry *newEntry = trie.lookup(vpn);
 101     if (newEntry) {
 102         assert(newEntry->vaddr == vpn);
 103         return newEntry;
 104     }
 105
 106     if (freeList.empty())
 107         evictLRU();
 108
 109     newEntry = freeList.front();
 110     freeList.pop_front();
 111
 112     *newEntry = entry;
 113     newEntry->lruSeq = nextSeq();
 114     newEntry->vaddr = vpn;
 115     newEntry->trieHandle =
 116     trie.insert(vpn, TlbEntryTrie::MaxBits - entry.logBytes, newEntry);
 117     return newEntry;
 118 }
 119
 120 TlbEntry *
 121 TLB::lookup(Addr va, bool update_lru)
 122 {
 123     TlbEntry *entry = trie.lookup(va);
 124     if (entry && update_lru)
 125         entry->lruSeq = nextSeq();
 126     return entry;
 127 }
 128
 129 void
 130 TLB::flushAll()
 131 {
 132     DPRINTF(TLB, "Invalidating all entries.\n");
 133     for (unsigned i = 0; i < size; i++) {
 134         if (tlb[i].trieHandle) {
 135             trie.remove(tlb[i].trieHandle);
 136             tlb[i].trieHandle = NULL;
 137             freeList.push_back(&tlb[i]);
 138         }
 139     }
 140 }
 141
 142 void
 143 TLB::setConfigAddress(uint32_t addr)
 144 {
 145     configAddress = addr;
 146 }
 147
 148 void
 149 TLB::flushNonGlobal()
 150 {
 151     DPRINTF(TLB, "Invalidating all non global entries.\n");
 152     for (unsigned i = 0; i < size; i++) {
 153         if (tlb[i].trieHandle && !tlb[i].global) {
 154             trie.remove(tlb[i].trieHandle);
 155             tlb[i].trieHandle = NULL;
 156             freeList.push_back(&tlb[i]);
 157         }
 158     }
 159 }
 160
 161 void
 162 TLB::demapPage(Addr va, uint64_t asn)
 163 {
 164     TlbEntry *entry = trie.lookup(va);
 165     if (entry) {
 166         trie.remove(entry->trieHandle);
 167         entry->trieHandle = NULL;
 168         freeList.push_back(entry);
 169     }
 170 }
 171
 172 namespace
 173 {
 174
 175 Cycles
 176 localMiscRegAccess(bool read, MiscRegIndex regNum,
 177                    ThreadContext *tc, PacketPtr pkt)
 178 {
 179     if (read) {
 180         RegVal data = htole(tc->readMiscReg(regNum));
 181         assert(pkt->getSize() <= sizeof(RegVal));
 182         pkt->setData((uint8_t *)&data);
 183     } else {
 184         RegVal data = htole(tc->readMiscRegNoEffect(regNum));
 185         assert(pkt->getSize() <= sizeof(RegVal));
 186         pkt->writeData((uint8_t *)&data);
 187         tc->setMiscReg(regNum, letoh(data));
 188     }
 189     return Cycles(1);
 190 }
 191
 192 } // anonymous namespace
 193
 194 Fault
 195 TLB::translateInt(bool read, RequestPtr req, ThreadContext *tc)
 196 {
 197     DPRINTF(TLB, "Addresses references internal memory.\n");
 198     Addr vaddr = req->getVaddr();
 199     Addr prefix = (vaddr >> 3) & IntAddrPrefixMask;
 200     if (prefix == IntAddrPrefixCPUID) {
 201         panic("CPUID memory space not yet implemented!\n");
 202     } else if (prefix == IntAddrPrefixMSR) {
 203         vaddr = (vaddr >> 3) & ~IntAddrPrefixMask;
 204
 205         MiscRegIndex regNum;
 206         if (!msrAddrToIndex(regNum, vaddr))
 207             return std::make_shared<GeneralProtection>(0);
 208
 209         req->setLocalAccessor(
 210             [read,regNum](ThreadContext *tc, PacketPtr pkt)
 211             {
 212                 return localMiscRegAccess(read, regNum, tc, pkt);
 213             }
 214         );
 215
 216         return NoFault;
 217     } else if (prefix == IntAddrPrefixIO) {
 218         // TODO If CPL > IOPL or in virtual mode, check the I/O permission
 219         // bitmap in the TSS.
 220
 221         Addr IOPort = vaddr & ~IntAddrPrefixMask;
 222         // Make sure the address fits in the expected 16 bit IO address
 223         // space.
 224         assert(!(IOPort & ~0xFFFF));
 225         if (IOPort == 0xCF8 && req->getSize() == 4) {
 226             req->setLocalAccessor(
 227                 [read](ThreadContext *tc, PacketPtr pkt)
 228                 {
 229                     return localMiscRegAccess(
 230                             read, MISCREG_PCI_CONFIG_ADDRESS, tc, pkt);
 231                 }
 232             );
 233         } else if ((IOPort & ~mask(2)) == 0xCFC) {
 234             req->setFlags(Request::UNCACHEABLE | Request::STRICT_ORDER);
 235             Addr configAddress =
 236                 tc->readMiscRegNoEffect(MISCREG_PCI_CONFIG_ADDRESS);
 237             if (bits(configAddress, 31, 31)) {
 238                 req->setPaddr(PhysAddrPrefixPciConfig |
 239                         mbits(configAddress, 30, 2) |
 240                         (IOPort & mask(2)));
 241             } else {
 242                 req->setPaddr(PhysAddrPrefixIO | IOPort);
 243             }
 244         } else {
 245             req->setFlags(Request::UNCACHEABLE | Request::STRICT_ORDER);
 246             req->setPaddr(PhysAddrPrefixIO | IOPort);
 247         }
 248         return NoFault;
 249     } else {
 250         panic("Access to unrecognized internal address space %#x.\n",
 251                 prefix);
 252     }
 253 }
 254
 255 Fault
 256 TLB::finalizePhysical(const RequestPtr &req,
 257                       ThreadContext *tc, Mode mode) const
 258 {
 259     Addr paddr = req->getPaddr();
 260
 261     if (m5opRange.contains(paddr)) {
 262         req->setFlags(Request::STRICT_ORDER);
 263         uint8_t func;
 264         PseudoInst::decodeAddrOffset(paddr - m5opRange.start(), func);
 265         req->setLocalAccessor(
 266             [func, mode](ThreadContext *tc, PacketPtr pkt) -> Cycles
 267             {
 268                 uint64_t ret =
 269                     PseudoInst::pseudoInst<X86PseudoInstABI>(tc, func);
 270                 if (mode == Read)
 271                     pkt->setLE(ret);
 272                 return Cycles(1);
 273             }
 274         );
 275     } else if (FullSystem) {
 276         // Check for an access to the local APIC
 277         LocalApicBase localApicBase =
 278             tc->readMiscRegNoEffect(MISCREG_APIC_BASE);
 279         AddrRange apicRange(localApicBase.base * PageBytes,
 280                             (localApicBase.base + 1) * PageBytes);
 281
 282         if (apicRange.contains(paddr)) {
 283             // The Intel developer's manuals say the below restrictions apply,
 284             // but the linux kernel, because of a compiler optimization, breaks
 285             // them.
 286             /*
 287             // Check alignment
 288             if (paddr & ((32/8) - 1))
 289                 return new GeneralProtection(0);
 290             // Check access size
 291             if (req->getSize() != (32/8))
 292                 return new GeneralProtection(0);
 293             */
 294             // Force the access to be uncacheable.
 295             req->setFlags(Request::UNCACHEABLE | Request::STRICT_ORDER);
 296             req->setPaddr(x86LocalAPICAddress(tc->contextId(),
 297                                               paddr - apicRange.start()));
 298         }
 299     }
 300
 301     return NoFault;
 302 }
 303
 304 Fault
 305 TLB::translate(const RequestPtr &req,
 306         ThreadContext *tc, Translation *translation,
 307         Mode mode, bool &delayedResponse, bool timing)
 308 {
 309     Request::Flags flags = req->getFlags();
 310     int seg = flags & SegmentFlagMask;
 311     bool storeCheck = flags & (StoreCheck << FlagShift);
 312
 313     delayedResponse = false;
 314
 315     // If this is true, we're dealing with a request to a non-memory address
 316     // space.
 317     if (seg == SEGMENT_REG_MS) {
 318         return translateInt(mode == Read, req, tc);
 319     }
 320
 321     Addr vaddr = req->getVaddr();
 322     DPRINTF(TLB, "Translating vaddr %#x.\n", vaddr);
 323
 324     HandyM5Reg m5Reg = tc->readMiscRegNoEffect(MISCREG_M5_REG);
 325
 326     // If protected mode has been enabled...
 327     if (m5Reg.prot) {
 328         DPRINTF(TLB, "In protected mode.\n");
 329         // If we're not in 64-bit mode, do protection/limit checks
 330         if (m5Reg.mode != LongMode) {
 331             DPRINTF(TLB, "Not in long mode. Checking segment protection.\n");
 332             // Check for a NULL segment selector.
 333             if (!(seg == SEGMENT_REG_TSG || seg == SYS_SEGMENT_REG_IDTR ||
 334                         seg == SEGMENT_REG_HS || seg == SEGMENT_REG_LS)
 335                     && !tc->readMiscRegNoEffect(MISCREG_SEG_SEL(seg)))
 336                 return std::make_shared<GeneralProtection>(0);
 337             bool expandDown = false;
 338             SegAttr attr = tc->readMiscRegNoEffect(MISCREG_SEG_ATTR(seg));
 339             if (seg >= SEGMENT_REG_ES && seg <= SEGMENT_REG_HS) {
 340                 if (!attr.writable && (mode == Write || storeCheck))
 341                     return std::make_shared<GeneralProtection>(0);
 342                 if (!attr.readable && mode == Read)
 343                     return std::make_shared<GeneralProtection>(0);
 344                 expandDown = attr.expandDown;
 345
 346             }
 347             Addr base = tc->readMiscRegNoEffect(MISCREG_SEG_BASE(seg));
 348             Addr limit = tc->readMiscRegNoEffect(MISCREG_SEG_LIMIT(seg));
 349             bool sizeOverride = (flags & (AddrSizeFlagBit << FlagShift));
 350             unsigned logSize = sizeOverride ? (unsigned)m5Reg.altAddr
 351                                             : (unsigned)m5Reg.defAddr;
 352             int size = (1 << logSize) * 8;
 353             Addr offset = bits(vaddr - base, size - 1, 0);
 354             Addr endOffset = offset + req->getSize() - 1;
 355             if (expandDown) {
 356                 DPRINTF(TLB, "Checking an expand down segment.\n");
 357                 warn_once("Expand down segments are untested.\n");
 358                 if (offset <= limit || endOffset <= limit)
 359                     return std::make_shared<GeneralProtection>(0);
 360             } else {
 361                 if (offset > limit || endOffset > limit)
 362                     return std::make_shared<GeneralProtection>(0);
 363             }
 364         }
 365         if (m5Reg.submode != SixtyFourBitMode ||
 366                 (flags & (AddrSizeFlagBit << FlagShift)))
 367             vaddr &= mask(32);
 368         // If paging is enabled, do the translation.
 369         if (m5Reg.paging) {
 370             DPRINTF(TLB, "Paging enabled.\n");
 371             // The vaddr already has the segment base applied.
 372             TlbEntry *entry = lookup(vaddr);
 373             if (mode == Read) {
 374                 rdAccesses++;
 375             } else {
 376                 wrAccesses++;
 377             }
 378             if (!entry) {
 379                 DPRINTF(TLB, "Handling a TLB miss for "
 380                         "address %#x at pc %#x.\n",
 381                         vaddr, tc->instAddr());
 382                 if (mode == Read) {
 383                     rdMisses++;
 384                 } else {
 385                     wrMisses++;
 386                 }
 387                 if (FullSystem) {
 388                     Fault fault = walker->start(tc, translation, req, mode);
 389                     if (timing || fault != NoFault) {
 390                         // This gets ignored in atomic mode.
 391                         delayedResponse = true;
 392                         return fault;
 393                     }
 394                     entry = lookup(vaddr);
 395                     assert(entry);
 396                 } else {
 397                     Process *p = tc->getProcessPtr();
 398                     const EmulationPageTable::Entry *pte =
 399                         p->pTable->lookup(vaddr);
 400                     if (!pte && mode != Execute) {
 401                         // Check if we just need to grow the stack.
 402                         if (p->fixupStackFault(vaddr)) {
 403                             // If we did, lookup the entry for the new page.
 404                             pte = p->pTable->lookup(vaddr);
 405                         }
 406                     }
 407                     if (!pte) {
 408                         return std::make_shared<PageFault>(vaddr, true, mode,
 409                                                            true, false);
 410                     } else {
 411                         Addr alignedVaddr = p->pTable->pageAlign(vaddr);
 412                         DPRINTF(TLB, "Mapping %#x to %#x\n", alignedVaddr,
 413                                 pte->paddr);
 414                         entry = insert(alignedVaddr, TlbEntry(
 415                                 p->pTable->pid(), alignedVaddr, pte->paddr,
 416                                 pte->flags & EmulationPageTable::Uncacheable,
 417                                 pte->flags & EmulationPageTable::ReadOnly));
 418                     }
 419                     DPRINTF(TLB, "Miss was serviced.\n");
 420                 }
 421             }
 422
 423             DPRINTF(TLB, "Entry found with paddr %#x, "
 424                     "doing protection checks.\n", entry->paddr);
 425             // Do paging protection checks.
 426             bool inUser = (m5Reg.cpl == 3 &&
 427                     !(flags & (CPL0FlagBit << FlagShift)));
 428             CR0 cr0 = tc->readMiscRegNoEffect(MISCREG_CR0);
 429             bool badWrite = (!entry->writable && (inUser || cr0.wp));
 430             if ((inUser && !entry->user) || (mode == Write && badWrite)) {
 431                 // The page must have been present to get into the TLB in
 432                 // the first place. We'll assume the reserved bits are
 433                 // fine even though we're not checking them.
 434                 return std::make_shared<PageFault>(vaddr, true, mode, inUser,
 435                                                    false);
 436             }
 437             if (storeCheck && badWrite) {
 438                 // This would fault if this were a write, so return a page
 439                 // fault that reflects that happening.
 440                 return std::make_shared<PageFault>(vaddr, true, Write, inUser,
 441                                                    false);
 442             }
 443
 444             Addr paddr = entry->paddr | (vaddr & mask(entry->logBytes));
 445             DPRINTF(TLB, "Translated %#x -> %#x.\n", vaddr, paddr);
 446             req->setPaddr(paddr);
 447             if (entry->uncacheable)
 448                 req->setFlags(Request::UNCACHEABLE | Request::STRICT_ORDER);
 449         } else {
 450             //Use the address which already has segmentation applied.
 451             DPRINTF(TLB, "Paging disabled.\n");
 452             DPRINTF(TLB, "Translated %#x -> %#x.\n", vaddr, vaddr);
 453             req->setPaddr(vaddr);
 454         }
 455     } else {
 456         // Real mode
 457         DPRINTF(TLB, "In real mode.\n");
 458         DPRINTF(TLB, "Translated %#x -> %#x.\n", vaddr, vaddr);
 459         req->setPaddr(vaddr);
 460     }
 461
 462     return finalizePhysical(req, tc, mode);
 463 }
 464
 465 Fault
 466 TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode)
 467 {
 468     bool delayedResponse;
 469     return TLB::translate(req, tc, NULL, mode, delayedResponse, false);
 470 }
 471
 472 void
 473 TLB::translateTiming(const RequestPtr &req, ThreadContext *tc,
 474         Translation *translation, Mode mode)
 475 {
 476     bool delayedResponse;
 477     assert(translation);
 478     Fault fault =
 479         TLB::translate(req, tc, translation, mode, delayedResponse, true);
 480     if (!delayedResponse)
 481         translation->finish(fault, req, tc, mode);
 482     else
 483         translation->markDelayed();
 484 }
 485
 486 Walker *
 487 TLB::getWalker()
 488 {
 489     return walker;
 490 }
 491
 492 void
 493 TLB::regStats()
 494 {
 495     using namespace Stats;
 496     BaseTLB::regStats();
 497     rdAccesses
 498         .name(name() + ".rdAccesses")
 499         .desc("TLB accesses on read requests");
 500
 501     wrAccesses
 502         .name(name() + ".wrAccesses")
 503         .desc("TLB accesses on write requests");
 504
 505     rdMisses
 506         .name(name() + ".rdMisses")
 507         .desc("TLB misses on read requests");
 508
 509     wrMisses
 510         .name(name() + ".wrMisses")
 511         .desc("TLB misses on write requests");
 512
 513 }
 514
 515 void
 516 TLB::serialize(CheckpointOut &cp) const
 517 {
 518     // Only store the entries in use.
 519     uint32_t _size = size - freeList.size();
 520     SERIALIZE_SCALAR(_size);
 521     SERIALIZE_SCALAR(lruSeq);
 522
 523     uint32_t _count = 0;
 524     for (uint32_t x = 0; x < size; x++) {
 525         if (tlb[x].trieHandle != NULL)
 526             tlb[x].serializeSection(cp, csprintf("Entry%d", _count++));
 527     }
 528 }
 529
 530 void
 531 TLB::unserialize(CheckpointIn &cp)
 532 {
 533     // Do not allow to restore with a smaller tlb.
 534     uint32_t _size;
 535     UNSERIALIZE_SCALAR(_size);
 536     if (_size > size) {
 537         fatal("TLB size less than the one in checkpoint!");
 538     }
 539
 540     UNSERIALIZE_SCALAR(lruSeq);
 541
 542     for (uint32_t x = 0; x < _size; x++) {
 543         TlbEntry *newEntry = freeList.front();
 544         freeList.pop_front();
 545
 546         newEntry->unserializeSection(cp, csprintf("Entry%d", x));
 547         newEntry->trieHandle = trie.insert(newEntry->vaddr,
 548             TlbEntryTrie::MaxBits - newEntry->logBytes, newEntry);
 549     }
 550 }
 551
 552 Port *
 553 TLB::getTableWalkerPort()
 554 {
 555     return &walker->getPort("port");
 556 }
 557
 558 } // namespace X86ISA
 559
 560 X86ISA::TLB *
 561 X86TLBParams::create()
 562 {
 563     return new X86ISA::TLB(this);
 564 }