src/arch/x86/pagetable_walker.cc

   1 /*
   2  * Copyright (c) 2007 The Hewlett-Packard Development Company
   3  * All rights reserved.
   4  *
   5  * The license below extends only to copyright in the software and shall
   6  * not be construed as granting a license to any other intellectual
   7  * property including but not limited to intellectual property relating
   8  * to a hardware implementation of the functionality of the software
   9  * licensed hereunder.  You may use the software subject to the license
  10  * terms below provided that you ensure that this notice is replicated
  11  * unmodified and in its entirety in all distributions of the software,
  12  * modified or unmodified, in source code or in binary form.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met: redistributions of source code must retain the above copyright
  17  * notice, this list of conditions and the following disclaimer;
  18  * redistributions in binary form must reproduce the above copyright
  19  * notice, this list of conditions and the following disclaimer in the
  20  * documentation and/or other materials provided with the distribution;
  21  * neither the name of the copyright holders nor the names of its
  22  * contributors may be used to endorse or promote products derived from
  23  * this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  *
  37  * Authors: Gabe Black
  38  */
  39
  40 #include "arch/x86/pagetable.hh"
  41 #include "arch/x86/pagetable_walker.hh"
  42 #include "arch/x86/tlb.hh"
  43 #include "base/bitfield.hh"
  44 #include "cpu/thread_context.hh"
  45 #include "cpu/base.hh"
  46 #include "mem/packet_access.hh"
  47 #include "mem/request.hh"
  48 #include "sim/system.hh"
  49
  50 namespace X86ISA {
  51
  52 // Unfortunately, the placement of the base field in a page table entry is
  53 // very erratic and would make a mess here. It might be moved here at some
  54 // point in the future.
  55 BitUnion64(PageTableEntry)
  56     Bitfield<63> nx;
  57     Bitfield<11, 9> avl;
  58     Bitfield<8> g;
  59     Bitfield<7> ps;
  60     Bitfield<6> d;
  61     Bitfield<5> a;
  62     Bitfield<4> pcd;
  63     Bitfield<3> pwt;
  64     Bitfield<2> u;
  65     Bitfield<1> w;
  66     Bitfield<0> p;
  67 EndBitUnion(PageTableEntry)
  68
  69 Fault
  70 Walker::doNext(PacketPtr &write)
  71 {
  72     assert(state != Ready && state != Waiting);
  73     write = NULL;
  74     PageTableEntry pte;
  75     if (size == 8)
  76         pte = read->get<uint64_t>();
  77     else
  78         pte = read->get<uint32_t>();
  79     VAddr vaddr = entry.vaddr;
  80     bool uncacheable = pte.pcd;
  81     Addr nextRead = 0;
  82     bool doWrite = false;
  83     bool badNX = pte.nx && mode == BaseTLB::Execute && enableNX;
  84     switch(state) {
  85       case LongPML4:
  86         DPRINTF(PageTableWalker,
  87                 "Got long mode PML4 entry %#016x.\n", (uint64_t)pte);
  88         nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl3 * size;
  89         doWrite = !pte.a;
  90         pte.a = 1;
  91         entry.writable = pte.w;
  92         entry.user = pte.u;
  93         if (badNX || !pte.p) {
  94             stop();
  95             return pageFault(pte.p);
  96         }
  97         entry.noExec = pte.nx;
  98         nextState = LongPDP;
  99         break;
 100       case LongPDP:
 101         DPRINTF(PageTableWalker,
 102                 "Got long mode PDP entry %#016x.\n", (uint64_t)pte);
 103         nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl2 * size;
 104         doWrite = !pte.a;
 105         pte.a = 1;
 106         entry.writable = entry.writable && pte.w;
 107         entry.user = entry.user && pte.u;
 108         if (badNX || !pte.p) {
 109             stop();
 110             return pageFault(pte.p);
 111         }
 112         nextState = LongPD;
 113         break;
 114       case LongPD:
 115         DPRINTF(PageTableWalker,
 116                 "Got long mode PD entry %#016x.\n", (uint64_t)pte);
 117         doWrite = !pte.a;
 118         pte.a = 1;
 119         entry.writable = entry.writable && pte.w;
 120         entry.user = entry.user && pte.u;
 121         if (badNX || !pte.p) {
 122             stop();
 123             return pageFault(pte.p);
 124         }
 125         if (!pte.ps) {
 126             // 4 KB page
 127             entry.size = 4 * (1 << 10);
 128             nextRead =
 129                 ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl1 * size;
 130             nextState = LongPTE;
 131             break;
 132         } else {
 133             // 2 MB page
 134             entry.size = 2 * (1 << 20);
 135             entry.paddr = (uint64_t)pte & (mask(31) << 21);
 136             entry.uncacheable = uncacheable;
 137             entry.global = pte.g;
 138             entry.patBit = bits(pte, 12);
 139             entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1);
 140             tlb->insert(entry.vaddr, entry);
 141             stop();
 142             return NoFault;
 143         }
 144       case LongPTE:
 145         DPRINTF(PageTableWalker,
 146                 "Got long mode PTE entry %#016x.\n", (uint64_t)pte);
 147         doWrite = !pte.a;
 148         pte.a = 1;
 149         entry.writable = entry.writable && pte.w;
 150         entry.user = entry.user && pte.u;
 151         if (badNX || !pte.p) {
 152             stop();
 153             return pageFault(pte.p);
 154         }
 155         entry.paddr = (uint64_t)pte & (mask(40) << 12);
 156         entry.uncacheable = uncacheable;
 157         entry.global = pte.g;
 158         entry.patBit = bits(pte, 12);
 159         entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
 160         tlb->insert(entry.vaddr, entry);
 161         stop();
 162         return NoFault;
 163       case PAEPDP:
 164         DPRINTF(PageTableWalker,
 165                 "Got legacy mode PAE PDP entry %#08x.\n", (uint32_t)pte);
 166         nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael2 * size;
 167         if (!pte.p) {
 168             stop();
 169             return pageFault(pte.p);
 170         }
 171         nextState = PAEPD;
 172         break;
 173       case PAEPD:
 174         DPRINTF(PageTableWalker,
 175                 "Got legacy mode PAE PD entry %#08x.\n", (uint32_t)pte);
 176         doWrite = !pte.a;
 177         pte.a = 1;
 178         entry.writable = pte.w;
 179         entry.user = pte.u;
 180         if (badNX || !pte.p) {
 181             stop();
 182             return pageFault(pte.p);
 183         }
 184         if (!pte.ps) {
 185             // 4 KB page
 186             entry.size = 4 * (1 << 10);
 187             nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael1 * size;
 188             nextState = PAEPTE;
 189             break;
 190         } else {
 191             // 2 MB page
 192             entry.size = 2 * (1 << 20);
 193             entry.paddr = (uint64_t)pte & (mask(31) << 21);
 194             entry.uncacheable = uncacheable;
 195             entry.global = pte.g;
 196             entry.patBit = bits(pte, 12);
 197             entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1);
 198             tlb->insert(entry.vaddr, entry);
 199             stop();
 200             return NoFault;
 201         }
 202       case PAEPTE:
 203         DPRINTF(PageTableWalker,
 204                 "Got legacy mode PAE PTE entry %#08x.\n", (uint32_t)pte);
 205         doWrite = !pte.a;
 206         pte.a = 1;
 207         entry.writable = entry.writable && pte.w;
 208         entry.user = entry.user && pte.u;
 209         if (badNX || !pte.p) {
 210             stop();
 211             return pageFault(pte.p);
 212         }
 213         entry.paddr = (uint64_t)pte & (mask(40) << 12);
 214         entry.uncacheable = uncacheable;
 215         entry.global = pte.g;
 216         entry.patBit = bits(pte, 7);
 217         entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
 218         tlb->insert(entry.vaddr, entry);
 219         stop();
 220         return NoFault;
 221       case PSEPD:
 222         DPRINTF(PageTableWalker,
 223                 "Got legacy mode PSE PD entry %#08x.\n", (uint32_t)pte);
 224         doWrite = !pte.a;
 225         pte.a = 1;
 226         entry.writable = pte.w;
 227         entry.user = pte.u;
 228         if (!pte.p) {
 229             stop();
 230             return pageFault(pte.p);
 231         }
 232         if (!pte.ps) {
 233             // 4 KB page
 234             entry.size = 4 * (1 << 10);
 235             nextRead =
 236                 ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * size;
 237             nextState = PTE;
 238             break;
 239         } else {
 240             // 4 MB page
 241             entry.size = 4 * (1 << 20);
 242             entry.paddr = bits(pte, 20, 13) << 32 | bits(pte, 31, 22) << 22;
 243             entry.uncacheable = uncacheable;
 244             entry.global = pte.g;
 245             entry.patBit = bits(pte, 12);
 246             entry.vaddr = entry.vaddr & ~((4 * (1 << 20)) - 1);
 247             tlb->insert(entry.vaddr, entry);
 248             stop();
 249             return NoFault;
 250         }
 251       case PD:
 252         DPRINTF(PageTableWalker,
 253                 "Got legacy mode PD entry %#08x.\n", (uint32_t)pte);
 254         doWrite = !pte.a;
 255         pte.a = 1;
 256         entry.writable = pte.w;
 257         entry.user = pte.u;
 258         if (!pte.p) {
 259             stop();
 260             return pageFault(pte.p);
 261         }
 262         // 4 KB page
 263         entry.size = 4 * (1 << 10);
 264         nextRead = ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * size;
 265         nextState = PTE;
 266         break;
 267       case PTE:
 268         DPRINTF(PageTableWalker,
 269                 "Got legacy mode PTE entry %#08x.\n", (uint32_t)pte);
 270         doWrite = !pte.a;
 271         pte.a = 1;
 272         entry.writable = pte.w;
 273         entry.user = pte.u;
 274         if (!pte.p) {
 275             stop();
 276             return pageFault(pte.p);
 277         }
 278         entry.paddr = (uint64_t)pte & (mask(20) << 12);
 279         entry.uncacheable = uncacheable;
 280         entry.global = pte.g;
 281         entry.patBit = bits(pte, 7);
 282         entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
 283         tlb->insert(entry.vaddr, entry);
 284         stop();
 285         return NoFault;
 286       default:
 287         panic("Unknown page table walker state %d!\n");
 288     }
 289     PacketPtr oldRead = read;
 290     //If we didn't return, we're setting up another read.
 291     Request::Flags flags = oldRead->req->getFlags();
 292     flags.set(Request::UNCACHEABLE, uncacheable);
 293     RequestPtr request =
 294         new Request(nextRead, oldRead->getSize(), flags);
 295     read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast);
 296     read->allocate();
 297     //If we need to write, adjust the read packet to write the modified value
 298     //back to memory.
 299     if (doWrite) {
 300         write = oldRead;
 301         write->set<uint64_t>(pte);
 302         write->cmd = MemCmd::WriteReq;
 303         write->setDest(Packet::Broadcast);
 304     } else {
 305         write = NULL;
 306         delete oldRead->req;
 307         delete oldRead;
 308     }
 309     return NoFault;
 310 }
 311
 312 Fault
 313 Walker::start(ThreadContext * _tc, BaseTLB::Translation *_translation,
 314               RequestPtr _req, BaseTLB::Mode _mode)
 315 {
 316     assert(state == Ready);
 317     tc = _tc;
 318     req = _req;
 319     Addr vaddr = req->getVaddr();
 320     mode = _mode;
 321     translation = _translation;
 322
 323     VAddr addr = vaddr;
 324
 325     //Figure out what we're doing.
 326     CR3 cr3 = tc->readMiscRegNoEffect(MISCREG_CR3);
 327     Addr top = 0;
 328     // Check if we're in long mode or not
 329     Efer efer = tc->readMiscRegNoEffect(MISCREG_EFER);
 330     size = 8;
 331     if (efer.lma) {
 332         // Do long mode.
 333         state = LongPML4;
 334         top = (cr3.longPdtb << 12) + addr.longl4 * size;
 335         enableNX = efer.nxe;
 336     } else {
 337         // We're in some flavor of legacy mode.
 338         CR4 cr4 = tc->readMiscRegNoEffect(MISCREG_CR4);
 339         if (cr4.pae) {
 340             // Do legacy PAE.
 341             state = PAEPDP;
 342             top = (cr3.paePdtb << 5) + addr.pael3 * size;
 343             enableNX = efer.nxe;
 344         } else {
 345             size = 4;
 346             top = (cr3.pdtb << 12) + addr.norml2 * size;
 347             if (cr4.pse) {
 348                 // Do legacy PSE.
 349                 state = PSEPD;
 350             } else {
 351                 // Do legacy non PSE.
 352                 state = PD;
 353             }
 354             enableNX = false;
 355         }
 356     }
 357
 358     nextState = Ready;
 359     entry.vaddr = vaddr;
 360
 361     Request::Flags flags = Request::PHYSICAL;
 362     if (cr3.pcd)
 363         flags.set(Request::UNCACHEABLE);
 364     RequestPtr request = new Request(top, size, flags);
 365     read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast);
 366     read->allocate();
 367     Enums::MemoryMode memMode = sys->getMemoryMode();
 368     if (memMode == Enums::timing) {
 369         nextState = state;
 370         state = Waiting;
 371         timingFault = NoFault;
 372         sendPackets();
 373     } else if (memMode == Enums::atomic) {
 374         Fault fault;
 375         do {
 376             port.sendAtomic(read);
 377             PacketPtr write = NULL;
 378             fault = doNext(write);
 379             assert(fault == NoFault || read == NULL);
 380             state = nextState;
 381             nextState = Ready;
 382             if (write)
 383                 port.sendAtomic(write);
 384         } while(read);
 385         state = Ready;
 386         nextState = Waiting;
 387         return fault;
 388     } else {
 389         panic("Unrecognized memory system mode.\n");
 390     }
 391     return NoFault;
 392 }
 393
 394 bool
 395 Walker::WalkerPort::recvTiming(PacketPtr pkt)
 396 {
 397     return walker->recvTiming(pkt);
 398 }
 399
 400 bool
 401 Walker::recvTiming(PacketPtr pkt)
 402 {
 403     if (pkt->isResponse() && !pkt->wasNacked()) {
 404         assert(inflight);
 405         assert(state == Waiting);
 406         assert(!read);
 407         inflight--;
 408         if (pkt->isRead()) {
 409             state = nextState;
 410             nextState = Ready;
 411             PacketPtr write = NULL;
 412             read = pkt;
 413             timingFault = doNext(write);
 414             state = Waiting;
 415             assert(timingFault == NoFault || read == NULL);
 416             if (write) {
 417                 writes.push_back(write);
 418             }
 419             sendPackets();
 420         } else {
 421             sendPackets();
 422         }
 423         if (inflight == 0 && read == NULL && writes.size() == 0) {
 424             state = Ready;
 425             nextState = Waiting;
 426             if (timingFault == NoFault) {
 427                 /*
 428                  * Finish the translation. Now that we now the right entry is
 429                  * in the TLB, this should work with no memory accesses.
 430                  * There could be new faults unrelated to the table walk like
 431                  * permissions violations, so we'll need the return value as
 432                  * well.
 433                  */
 434                 bool delayedResponse;
 435                 Fault fault = tlb->translate(req, tc, NULL, mode,
 436                         delayedResponse, true);
 437                 assert(!delayedResponse);
 438                 // Let the CPU continue.
 439                 translation->finish(fault, req, tc, mode);
 440             } else {
 441                 // There was a fault during the walk. Let the CPU know.
 442                 translation->finish(timingFault, req, tc, mode);
 443             }
 444         }
 445     } else if (pkt->wasNacked()) {
 446         pkt->reinitNacked();
 447         if (!port.sendTiming(pkt)) {
 448             inflight--;
 449             retrying = true;
 450             if (pkt->isWrite()) {
 451                 writes.push_back(pkt);
 452             } else {
 453                 assert(!read);
 454                 read = pkt;
 455             }
 456         }
 457     }
 458     return true;
 459 }
 460
 461 Tick
 462 Walker::WalkerPort::recvAtomic(PacketPtr pkt)
 463 {
 464     return 0;
 465 }
 466
 467 void
 468 Walker::WalkerPort::recvFunctional(PacketPtr pkt)
 469 {
 470     return;
 471 }
 472
 473 void
 474 Walker::WalkerPort::recvStatusChange(Status status)
 475 {
 476     if (status == RangeChange) {
 477         if (!snoopRangeSent) {
 478             snoopRangeSent = true;
 479             sendStatusChange(Port::RangeChange);
 480         }
 481         return;
 482     }
 483
 484     panic("Unexpected recvStatusChange.\n");
 485 }
 486
 487 void
 488 Walker::WalkerPort::recvRetry()
 489 {
 490     walker->recvRetry();
 491 }
 492
 493 void
 494 Walker::recvRetry()
 495 {
 496     retrying = false;
 497     sendPackets();
 498 }
 499
 500 void
 501 Walker::sendPackets()
 502 {
 503     //If we're already waiting for the port to become available, just return.
 504     if (retrying)
 505         return;
 506
 507     //Reads always have priority
 508     if (read) {
 509         PacketPtr pkt = read;
 510         read = NULL;
 511         inflight++;
 512         if (!port.sendTiming(pkt)) {
 513             retrying = true;
 514             read = pkt;
 515             inflight--;
 516             return;
 517         }
 518     }
 519     //Send off as many of the writes as we can.
 520     while (writes.size()) {
 521         PacketPtr write = writes.back();
 522         writes.pop_back();
 523         inflight++;
 524         if (!port.sendTiming(write)) {
 525             retrying = true;
 526             writes.push_back(write);
 527             inflight--;
 528             return;
 529         }
 530     }
 531 }
 532
 533 Port *
 534 Walker::getPort(const std::string &if_name, int idx)
 535 {
 536     if (if_name == "port")
 537         return &port;
 538     else
 539         panic("No page table walker port named %s!\n", if_name);
 540 }
 541
 542 Fault
 543 Walker::pageFault(bool present)
 544 {
 545     DPRINTF(PageTableWalker, "Raising page fault.\n");
 546     HandyM5Reg m5reg = tc->readMiscRegNoEffect(MISCREG_M5_REG);
 547     if (mode == BaseTLB::Execute && !enableNX)
 548         mode = BaseTLB::Read;
 549     return new PageFault(entry.vaddr, present, mode, m5reg.cpl == 3, false);
 550 }
 551
 552 }
 553
 554 X86ISA::Walker *
 555 X86PagetableWalkerParams::create()
 556 {
 557     return new X86ISA::Walker(this);
 558 }