src/arch/x86/pagetable_walker.cc

   1 /*
   2  * Copyright (c) 2007 The Hewlett-Packard Development Company
   3  * All rights reserved.
   4  *
   5  * Redistribution and use of this software in source and binary forms,
   6  * with or without modification, are permitted provided that the
   7  * following conditions are met:
   8  *
   9  * The software must be used only for Non-Commercial Use which means any
  10  * use which is NOT directed to receiving any direct monetary
  11  * compensation for, or commercial advantage from such use.  Illustrative
  12  * examples of non-commercial use are academic research, personal study,
  13  * teaching, education and corporate research & development.
  14  * Illustrative examples of commercial use are distributing products for
  15  * commercial advantage and providing services using the software for
  16  * commercial advantage.
  17  *
  18  * If you wish to use this software or functionality therein that may be
  19  * covered by patents for commercial use, please contact:
  20  *     Director of Intellectual Property Licensing
  21  *     Office of Strategy and Technology
  22  *     Hewlett-Packard Company
  23  *     1501 Page Mill Road
  24  *     Palo Alto, California  94304
  25  *
  26  * Redistributions of source code must retain the above copyright notice,
  27  * this list of conditions and the following disclaimer.  Redistributions
  28  * in binary form must reproduce the above copyright notice, this list of
  29  * conditions and the following disclaimer in the documentation and/or
  30  * other materials provided with the distribution.  Neither the name of
  31  * the COPYRIGHT HOLDER(s), HEWLETT-PACKARD COMPANY, nor the names of its
  32  * contributors may be used to endorse or promote products derived from
  33  * this software without specific prior written permission.  No right of
  34  * sublicense is granted herewith.  Derivatives of the software and
  35  * output created using the software may be prepared, but only for
  36  * Non-Commercial Uses.  Derivatives of the software may be shared with
  37  * others provided: (i) the others agree to abide by the list of
  38  * conditions herein which includes the Non-Commercial Use restrictions;
  39  * and (ii) such Derivatives of the software include the above copyright
  40  * notice to acknowledge the contribution from this software where
  41  * applicable, this list of conditions and the disclaimer below.
  42  *
  43  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  44  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  45  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  46  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  47  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  48  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  49  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  50  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  51  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  52  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  53  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  54  *
  55  * Authors: Gabe Black
  56  */
  57
  58 #include "arch/x86/pagetable.hh"
  59 #include "arch/x86/pagetable_walker.hh"
  60 #include "arch/x86/tlb.hh"
  61 #include "base/bitfield.hh"
  62 #include "cpu/thread_context.hh"
  63 #include "cpu/base.hh"
  64 #include "mem/packet_access.hh"
  65 #include "mem/request.hh"
  66 #include "sim/system.hh"
  67
  68 namespace X86ISA {
  69
  70 // Unfortunately, the placement of the base field in a page table entry is
  71 // very erratic and would make a mess here. It might be moved here at some
  72 // point in the future.
  73 BitUnion64(PageTableEntry)
  74     Bitfield<63> nx;
  75     Bitfield<11, 9> avl;
  76     Bitfield<8> g;
  77     Bitfield<7> ps;
  78     Bitfield<6> d;
  79     Bitfield<5> a;
  80     Bitfield<4> pcd;
  81     Bitfield<3> pwt;
  82     Bitfield<2> u;
  83     Bitfield<1> w;
  84     Bitfield<0> p;
  85 EndBitUnion(PageTableEntry)
  86
  87 Fault
  88 Walker::doNext(PacketPtr &write)
  89 {
  90     assert(state != Ready && state != Waiting);
  91     write = NULL;
  92     PageTableEntry pte;
  93     if (size == 8)
  94         pte = read->get<uint64_t>();
  95     else
  96         pte = read->get<uint32_t>();
  97     VAddr vaddr = entry.vaddr;
  98     bool uncacheable = pte.pcd;
  99     Addr nextRead = 0;
 100     bool doWrite = false;
 101     bool badNX = pte.nx && mode == BaseTLB::Execute && enableNX;
 102     switch(state) {
 103       case LongPML4:
 104         DPRINTF(PageTableWalker,
 105                 "Got long mode PML4 entry %#016x.\n", (uint64_t)pte);
 106         nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl3 * size;
 107         doWrite = !pte.a;
 108         pte.a = 1;
 109         entry.writable = pte.w;
 110         entry.user = pte.u;
 111         if (badNX || !pte.p) {
 112             stop();
 113             return pageFault(pte.p);
 114         }
 115         entry.noExec = pte.nx;
 116         nextState = LongPDP;
 117         break;
 118       case LongPDP:
 119         DPRINTF(PageTableWalker,
 120                 "Got long mode PDP entry %#016x.\n", (uint64_t)pte);
 121         nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl2 * size;
 122         doWrite = !pte.a;
 123         pte.a = 1;
 124         entry.writable = entry.writable && pte.w;
 125         entry.user = entry.user && pte.u;
 126         if (badNX || !pte.p) {
 127             stop();
 128             return pageFault(pte.p);
 129         }
 130         nextState = LongPD;
 131         break;
 132       case LongPD:
 133         DPRINTF(PageTableWalker,
 134                 "Got long mode PD entry %#016x.\n", (uint64_t)pte);
 135         doWrite = !pte.a;
 136         pte.a = 1;
 137         entry.writable = entry.writable && pte.w;
 138         entry.user = entry.user && pte.u;
 139         if (badNX || !pte.p) {
 140             stop();
 141             return pageFault(pte.p);
 142         }
 143         if (!pte.ps) {
 144             // 4 KB page
 145             entry.size = 4 * (1 << 10);
 146             nextRead =
 147                 ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl1 * size;
 148             nextState = LongPTE;
 149             break;
 150         } else {
 151             // 2 MB page
 152             entry.size = 2 * (1 << 20);
 153             entry.paddr = (uint64_t)pte & (mask(31) << 21);
 154             entry.uncacheable = uncacheable;
 155             entry.global = pte.g;
 156             entry.patBit = bits(pte, 12);
 157             entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1);
 158             tlb->insert(entry.vaddr, entry);
 159             stop();
 160             return NoFault;
 161         }
 162       case LongPTE:
 163         DPRINTF(PageTableWalker,
 164                 "Got long mode PTE entry %#016x.\n", (uint64_t)pte);
 165         doWrite = !pte.a;
 166         pte.a = 1;
 167         entry.writable = entry.writable && pte.w;
 168         entry.user = entry.user && pte.u;
 169         if (badNX || !pte.p) {
 170             stop();
 171             return pageFault(pte.p);
 172         }
 173         entry.paddr = (uint64_t)pte & (mask(40) << 12);
 174         entry.uncacheable = uncacheable;
 175         entry.global = pte.g;
 176         entry.patBit = bits(pte, 12);
 177         entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
 178         tlb->insert(entry.vaddr, entry);
 179         stop();
 180         return NoFault;
 181       case PAEPDP:
 182         DPRINTF(PageTableWalker,
 183                 "Got legacy mode PAE PDP entry %#08x.\n", (uint32_t)pte);
 184         nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael2 * size;
 185         if (!pte.p) {
 186             stop();
 187             return pageFault(pte.p);
 188         }
 189         nextState = PAEPD;
 190         break;
 191       case PAEPD:
 192         DPRINTF(PageTableWalker,
 193                 "Got legacy mode PAE PD entry %#08x.\n", (uint32_t)pte);
 194         doWrite = !pte.a;
 195         pte.a = 1;
 196         entry.writable = pte.w;
 197         entry.user = pte.u;
 198         if (badNX || !pte.p) {
 199             stop();
 200             return pageFault(pte.p);
 201         }
 202         if (!pte.ps) {
 203             // 4 KB page
 204             entry.size = 4 * (1 << 10);
 205             nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael1 * size;
 206             nextState = PAEPTE;
 207             break;
 208         } else {
 209             // 2 MB page
 210             entry.size = 2 * (1 << 20);
 211             entry.paddr = (uint64_t)pte & (mask(31) << 21);
 212             entry.uncacheable = uncacheable;
 213             entry.global = pte.g;
 214             entry.patBit = bits(pte, 12);
 215             entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1);
 216             tlb->insert(entry.vaddr, entry);
 217             stop();
 218             return NoFault;
 219         }
 220       case PAEPTE:
 221         DPRINTF(PageTableWalker,
 222                 "Got legacy mode PAE PTE entry %#08x.\n", (uint32_t)pte);
 223         doWrite = !pte.a;
 224         pte.a = 1;
 225         entry.writable = entry.writable && pte.w;
 226         entry.user = entry.user && pte.u;
 227         if (badNX || !pte.p) {
 228             stop();
 229             return pageFault(pte.p);
 230         }
 231         entry.paddr = (uint64_t)pte & (mask(40) << 12);
 232         entry.uncacheable = uncacheable;
 233         entry.global = pte.g;
 234         entry.patBit = bits(pte, 7);
 235         entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
 236         tlb->insert(entry.vaddr, entry);
 237         stop();
 238         return NoFault;
 239       case PSEPD:
 240         DPRINTF(PageTableWalker,
 241                 "Got legacy mode PSE PD entry %#08x.\n", (uint32_t)pte);
 242         doWrite = !pte.a;
 243         pte.a = 1;
 244         entry.writable = pte.w;
 245         entry.user = pte.u;
 246         if (!pte.p) {
 247             stop();
 248             return pageFault(pte.p);
 249         }
 250         if (!pte.ps) {
 251             // 4 KB page
 252             entry.size = 4 * (1 << 10);
 253             nextRead =
 254                 ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * size;
 255             nextState = PTE;
 256             break;
 257         } else {
 258             // 4 MB page
 259             entry.size = 4 * (1 << 20);
 260             entry.paddr = bits(pte, 20, 13) << 32 | bits(pte, 31, 22) << 22;
 261             entry.uncacheable = uncacheable;
 262             entry.global = pte.g;
 263             entry.patBit = bits(pte, 12);
 264             entry.vaddr = entry.vaddr & ~((4 * (1 << 20)) - 1);
 265             tlb->insert(entry.vaddr, entry);
 266             stop();
 267             return NoFault;
 268         }
 269       case PD:
 270         DPRINTF(PageTableWalker,
 271                 "Got legacy mode PD entry %#08x.\n", (uint32_t)pte);
 272         doWrite = !pte.a;
 273         pte.a = 1;
 274         entry.writable = pte.w;
 275         entry.user = pte.u;
 276         if (!pte.p) {
 277             stop();
 278             return pageFault(pte.p);
 279         }
 280         // 4 KB page
 281         entry.size = 4 * (1 << 10);
 282         nextRead = ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * size;
 283         nextState = PTE;
 284         break;
 285       case PTE:
 286         DPRINTF(PageTableWalker,
 287                 "Got legacy mode PTE entry %#08x.\n", (uint32_t)pte);
 288         doWrite = !pte.a;
 289         pte.a = 1;
 290         entry.writable = pte.w;
 291         entry.user = pte.u;
 292         if (!pte.p) {
 293             stop();
 294             return pageFault(pte.p);
 295         }
 296         entry.paddr = (uint64_t)pte & (mask(20) << 12);
 297         entry.uncacheable = uncacheable;
 298         entry.global = pte.g;
 299         entry.patBit = bits(pte, 7);
 300         entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
 301         tlb->insert(entry.vaddr, entry);
 302         stop();
 303         return NoFault;
 304       default:
 305         panic("Unknown page table walker state %d!\n");
 306     }
 307     PacketPtr oldRead = read;
 308     //If we didn't return, we're setting up another read.
 309     Request::Flags flags = oldRead->req->getFlags();
 310     flags.set(Request::UNCACHEABLE, uncacheable);
 311     RequestPtr request =
 312         new Request(nextRead, oldRead->getSize(), flags);
 313     read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast);
 314     read->allocate();
 315     //If we need to write, adjust the read packet to write the modified value
 316     //back to memory.
 317     if (doWrite) {
 318         write = oldRead;
 319         write->set<uint64_t>(pte);
 320         write->cmd = MemCmd::WriteReq;
 321         write->setDest(Packet::Broadcast);
 322     } else {
 323         write = NULL;
 324         delete oldRead->req;
 325         delete oldRead;
 326     }
 327     return NoFault;
 328 }
 329
 330 Fault
 331 Walker::start(ThreadContext * _tc, BaseTLB::Translation *_translation,
 332               RequestPtr _req, BaseTLB::Mode _mode)
 333 {
 334     assert(state == Ready);
 335     tc = _tc;
 336     req = _req;
 337     Addr vaddr = req->getVaddr();
 338     mode = _mode;
 339     translation = _translation;
 340
 341     VAddr addr = vaddr;
 342
 343     //Figure out what we're doing.
 344     CR3 cr3 = tc->readMiscRegNoEffect(MISCREG_CR3);
 345     Addr top = 0;
 346     // Check if we're in long mode or not
 347     Efer efer = tc->readMiscRegNoEffect(MISCREG_EFER);
 348     size = 8;
 349     if (efer.lma) {
 350         // Do long mode.
 351         state = LongPML4;
 352         top = (cr3.longPdtb << 12) + addr.longl4 * size;
 353         enableNX = efer.nxe;
 354     } else {
 355         // We're in some flavor of legacy mode.
 356         CR4 cr4 = tc->readMiscRegNoEffect(MISCREG_CR4);
 357         if (cr4.pae) {
 358             // Do legacy PAE.
 359             state = PAEPDP;
 360             top = (cr3.paePdtb << 5) + addr.pael3 * size;
 361             enableNX = efer.nxe;
 362         } else {
 363             size = 4;
 364             top = (cr3.pdtb << 12) + addr.norml2 * size;
 365             if (cr4.pse) {
 366                 // Do legacy PSE.
 367                 state = PSEPD;
 368             } else {
 369                 // Do legacy non PSE.
 370                 state = PD;
 371             }
 372             enableNX = false;
 373         }
 374     }
 375
 376     nextState = Ready;
 377     entry.vaddr = vaddr;
 378
 379     Request::Flags flags = Request::PHYSICAL;
 380     if (cr3.pcd)
 381         flags.set(Request::UNCACHEABLE);
 382     RequestPtr request = new Request(top, size, flags);
 383     read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast);
 384     read->allocate();
 385     Enums::MemoryMode memMode = sys->getMemoryMode();
 386     if (memMode == Enums::timing) {
 387         nextState = state;
 388         state = Waiting;
 389         timingFault = NoFault;
 390         sendPackets();
 391     } else if (memMode == Enums::atomic) {
 392         Fault fault;
 393         do {
 394             port.sendAtomic(read);
 395             PacketPtr write = NULL;
 396             fault = doNext(write);
 397             assert(fault == NoFault || read == NULL);
 398             state = nextState;
 399             nextState = Ready;
 400             if (write)
 401                 port.sendAtomic(write);
 402         } while(read);
 403         state = Ready;
 404         nextState = Waiting;
 405         return fault;
 406     } else {
 407         panic("Unrecognized memory system mode.\n");
 408     }
 409     return NoFault;
 410 }
 411
 412 bool
 413 Walker::WalkerPort::recvTiming(PacketPtr pkt)
 414 {
 415     return walker->recvTiming(pkt);
 416 }
 417
 418 bool
 419 Walker::recvTiming(PacketPtr pkt)
 420 {
 421     if (pkt->isResponse() && !pkt->wasNacked()) {
 422         assert(inflight);
 423         assert(state == Waiting);
 424         assert(!read);
 425         inflight--;
 426         if (pkt->isRead()) {
 427             state = nextState;
 428             nextState = Ready;
 429             PacketPtr write = NULL;
 430             read = pkt;
 431             timingFault = doNext(write);
 432             state = Waiting;
 433             assert(timingFault == NoFault || read == NULL);
 434             if (write) {
 435                 writes.push_back(write);
 436             }
 437             sendPackets();
 438         } else {
 439             sendPackets();
 440         }
 441         if (inflight == 0 && read == NULL && writes.size() == 0) {
 442             state = Ready;
 443             nextState = Waiting;
 444             if (timingFault == NoFault) {
 445                 /*
 446                  * Finish the translation. Now that we now the right entry is
 447                  * in the TLB, this should work with no memory accesses.
 448                  * There could be new faults unrelated to the table walk like
 449                  * permissions violations, so we'll need the return value as
 450                  * well.
 451                  */
 452                 bool delayedResponse;
 453                 Fault fault = tlb->translate(req, tc, NULL, mode,
 454                         delayedResponse, true);
 455                 assert(!delayedResponse);
 456                 // Let the CPU continue.
 457                 translation->finish(fault, req, tc, mode);
 458             } else {
 459                 // There was a fault during the walk. Let the CPU know.
 460                 translation->finish(timingFault, req, tc, mode);
 461             }
 462         }
 463     } else if (pkt->wasNacked()) {
 464         pkt->reinitNacked();
 465         if (!port.sendTiming(pkt)) {
 466             inflight--;
 467             retrying = true;
 468             if (pkt->isWrite()) {
 469                 writes.push_back(pkt);
 470             } else {
 471                 assert(!read);
 472                 read = pkt;
 473             }
 474         }
 475     }
 476     return true;
 477 }
 478
 479 Tick
 480 Walker::WalkerPort::recvAtomic(PacketPtr pkt)
 481 {
 482     return 0;
 483 }
 484
 485 void
 486 Walker::WalkerPort::recvFunctional(PacketPtr pkt)
 487 {
 488     return;
 489 }
 490
 491 void
 492 Walker::WalkerPort::recvStatusChange(Status status)
 493 {
 494     if (status == RangeChange) {
 495         if (!snoopRangeSent) {
 496             snoopRangeSent = true;
 497             sendStatusChange(Port::RangeChange);
 498         }
 499         return;
 500     }
 501
 502     panic("Unexpected recvStatusChange.\n");
 503 }
 504
 505 void
 506 Walker::WalkerPort::recvRetry()
 507 {
 508     walker->recvRetry();
 509 }
 510
 511 void
 512 Walker::recvRetry()
 513 {
 514     retrying = false;
 515     sendPackets();
 516 }
 517
 518 void
 519 Walker::sendPackets()
 520 {
 521     //If we're already waiting for the port to become available, just return.
 522     if (retrying)
 523         return;
 524
 525     //Reads always have priority
 526     if (read) {
 527         PacketPtr pkt = read;
 528         read = NULL;
 529         inflight++;
 530         if (!port.sendTiming(pkt)) {
 531             retrying = true;
 532             read = pkt;
 533             inflight--;
 534             return;
 535         }
 536     }
 537     //Send off as many of the writes as we can.
 538     while (writes.size()) {
 539         PacketPtr write = writes.back();
 540         writes.pop_back();
 541         inflight++;
 542         if (!port.sendTiming(write)) {
 543             retrying = true;
 544             writes.push_back(write);
 545             inflight--;
 546             return;
 547         }
 548     }
 549 }
 550
 551 Port *
 552 Walker::getPort(const std::string &if_name, int idx)
 553 {
 554     if (if_name == "port")
 555         return &port;
 556     else
 557         panic("No page table walker port named %s!\n", if_name);
 558 }
 559
 560 Fault
 561 Walker::pageFault(bool present)
 562 {
 563     DPRINTF(PageTableWalker, "Raising page fault.\n");
 564     HandyM5Reg m5reg = tc->readMiscRegNoEffect(MISCREG_M5_REG);
 565     if (mode == BaseTLB::Execute && !enableNX)
 566         mode = BaseTLB::Read;
 567     return new PageFault(entry.vaddr, present, mode, m5reg.cpl == 3, false);
 568 }
 569
 570 }
 571
 572 X86ISA::Walker *
 573 X86PagetableWalkerParams::create()
 574 {
 575     return new X86ISA::Walker(this);
 576 }