src/arch/riscv/pagetable_walker.cc

   1 /*
   2  * Copyright (c) 2012 ARM Limited
   3  * Copyright (c) 2020 Barkhausen Institut
   4  * All rights reserved.
   5  *
   6  * The license below extends only to copyright in the software and shall
   7  * not be construed as granting a license to any other intellectual
   8  * property including but not limited to intellectual property relating
   9  * to a hardware implementation of the functionality of the software
  10  * licensed hereunder.  You may use the software subject to the license
  11  * terms below provided that you ensure that this notice is replicated
  12  * unmodified and in its entirety in all distributions of the software,
  13  * modified or unmodified, in source code or in binary form.
  14  *
  15  * Copyright (c) 2007 The Hewlett-Packard Development Company
  16  * All rights reserved.
  17  *
  18  * The license below extends only to copyright in the software and shall
  19  * not be construed as granting a license to any other intellectual
  20  * property including but not limited to intellectual property relating
  21  * to a hardware implementation of the functionality of the software
  22  * licensed hereunder.  You may use the software subject to the license
  23  * terms below provided that you ensure that this notice is replicated
  24  * unmodified and in its entirety in all distributions of the software,
  25  * modified or unmodified, in source code or in binary form.
  26  *
  27  * Redistribution and use in source and binary forms, with or without
  28  * modification, are permitted provided that the following conditions are
  29  * met: redistributions of source code must retain the above copyright
  30  * notice, this list of conditions and the following disclaimer;
  31  * redistributions in binary form must reproduce the above copyright
  32  * notice, this list of conditions and the following disclaimer in the
  33  * documentation and/or other materials provided with the distribution;
  34  * neither the name of the copyright holders nor the names of its
  35  * contributors may be used to endorse or promote products derived from
  36  * this software without specific prior written permission.
  37  *
  38  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  39  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  40  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  41  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  42  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  43  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  44  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  45  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  46  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  47  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  48  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  49  */
  50
  51 #include "arch/riscv/pagetable_walker.hh"
  52
  53 #include <memory>
  54
  55 #include "arch/riscv/faults.hh"
  56 #include "arch/riscv/pagetable.hh"
  57 #include "arch/riscv/tlb.hh"
  58 #include "base/bitfield.hh"
  59 #include "base/trie.hh"
  60 #include "cpu/base.hh"
  61 #include "cpu/thread_context.hh"
  62 #include "debug/PageTableWalker.hh"
  63 #include "mem/packet_access.hh"
  64 #include "mem/request.hh"
  65
  66 namespace RiscvISA {
  67
  68 Fault
  69 Walker::start(ThreadContext * _tc, BaseTLB::Translation *_translation,
  70               const RequestPtr &_req, BaseTLB::Mode _mode)
  71 {
  72     // TODO: in timing mode, instead of blocking when there are other
  73     // outstanding requests, see if this request can be coalesced with
  74     // another one (i.e. either coalesce or start walk)
  75     WalkerState * newState = new WalkerState(this, _translation, _req);
  76     newState->initState(_tc, _mode, sys->isTimingMode());
  77     if (currStates.size()) {
  78         assert(newState->isTiming());
  79         DPRINTF(PageTableWalker, "Walks in progress: %d\n", currStates.size());
  80         currStates.push_back(newState);
  81         return NoFault;
  82     } else {
  83         currStates.push_back(newState);
  84         Fault fault = newState->startWalk();
  85         if (!newState->isTiming()) {
  86             currStates.pop_front();
  87             delete newState;
  88         }
  89         return fault;
  90     }
  91 }
  92
  93 Fault
  94 Walker::startFunctional(ThreadContext * _tc, Addr &addr, unsigned &logBytes,
  95               BaseTLB::Mode _mode)
  96 {
  97     funcState.initState(_tc, _mode);
  98     return funcState.startFunctional(addr, logBytes);
  99 }
 100
 101 bool
 102 Walker::WalkerPort::recvTimingResp(PacketPtr pkt)
 103 {
 104     return walker->recvTimingResp(pkt);
 105 }
 106
 107 bool
 108 Walker::recvTimingResp(PacketPtr pkt)
 109 {
 110     WalkerSenderState * senderState =
 111         dynamic_cast<WalkerSenderState *>(pkt->popSenderState());
 112     WalkerState * senderWalk = senderState->senderWalk;
 113     bool walkComplete = senderWalk->recvPacket(pkt);
 114     delete senderState;
 115     if (walkComplete) {
 116         std::list<WalkerState *>::iterator iter;
 117         for (iter = currStates.begin(); iter != currStates.end(); iter++) {
 118             WalkerState * walkerState = *(iter);
 119             if (walkerState == senderWalk) {
 120                 iter = currStates.erase(iter);
 121                 break;
 122             }
 123         }
 124         delete senderWalk;
 125         // Since we block requests when another is outstanding, we
 126         // need to check if there is a waiting request to be serviced
 127         if (currStates.size() && !startWalkWrapperEvent.scheduled())
 128             // delay sending any new requests until we are finished
 129             // with the responses
 130             schedule(startWalkWrapperEvent, clockEdge());
 131     }
 132     return true;
 133 }
 134
 135 void
 136 Walker::WalkerPort::recvReqRetry()
 137 {
 138     walker->recvReqRetry();
 139 }
 140
 141 void
 142 Walker::recvReqRetry()
 143 {
 144     std::list<WalkerState *>::iterator iter;
 145     for (iter = currStates.begin(); iter != currStates.end(); iter++) {
 146         WalkerState * walkerState = *(iter);
 147         if (walkerState->isRetrying()) {
 148             walkerState->retry();
 149         }
 150     }
 151 }
 152
 153 bool Walker::sendTiming(WalkerState* sendingState, PacketPtr pkt)
 154 {
 155     WalkerSenderState* walker_state = new WalkerSenderState(sendingState);
 156     pkt->pushSenderState(walker_state);
 157     if (port.sendTimingReq(pkt)) {
 158         return true;
 159     } else {
 160         // undo the adding of the sender state and delete it, as we
 161         // will do it again the next time we attempt to send it
 162         pkt->popSenderState();
 163         delete walker_state;
 164         return false;
 165     }
 166
 167 }
 168
 169 Port &
 170 Walker::getPort(const std::string &if_name, PortID idx)
 171 {
 172     if (if_name == "port")
 173         return port;
 174     else
 175         return ClockedObject::getPort(if_name, idx);
 176 }
 177
 178 void
 179 Walker::WalkerState::initState(ThreadContext * _tc,
 180         BaseTLB::Mode _mode, bool _isTiming)
 181 {
 182     assert(state == Ready);
 183     started = false;
 184     tc = _tc;
 185     mode = _mode;
 186     timing = _isTiming;
 187     // fetch these now in case they change during the walk
 188     status = tc->readMiscReg(MISCREG_STATUS);
 189     pmode = walker->tlb->getMemPriv(tc, mode);
 190     satp = tc->readMiscReg(MISCREG_SATP);
 191     assert(satp.mode == AddrXlateMode::SV39);
 192 }
 193
 194 void
 195 Walker::startWalkWrapper()
 196 {
 197     unsigned num_squashed = 0;
 198     WalkerState *currState = currStates.front();
 199     while ((num_squashed < numSquashable) && currState &&
 200         currState->translation->squashed()) {
 201         currStates.pop_front();
 202         num_squashed++;
 203
 204         DPRINTF(PageTableWalker, "Squashing table walk for address %#x\n",
 205             currState->req->getVaddr());
 206
 207         // finish the translation which will delete the translation object
 208         currState->translation->finish(
 209             std::make_shared<UnimpFault>("Squashed Inst"),
 210             currState->req, currState->tc, currState->mode);
 211
 212         // delete the current request if there are no inflight packets.
 213         // if there is something in flight, delete when the packets are
 214         // received and inflight is zero.
 215         if (currState->numInflight() == 0) {
 216             delete currState;
 217         } else {
 218             currState->squash();
 219         }
 220
 221         // check the next translation request, if it exists
 222         if (currStates.size())
 223             currState = currStates.front();
 224         else
 225             currState = NULL;
 226     }
 227     if (currState && !currState->wasStarted())
 228         currState->startWalk();
 229 }
 230
 231 Fault
 232 Walker::WalkerState::startWalk()
 233 {
 234     Fault fault = NoFault;
 235     assert(!started);
 236     started = true;
 237     setupWalk(req->getVaddr());
 238     if (timing) {
 239         nextState = state;
 240         state = Waiting;
 241         timingFault = NoFault;
 242         sendPackets();
 243     } else {
 244         do {
 245             walker->port.sendAtomic(read);
 246             PacketPtr write = NULL;
 247             fault = stepWalk(write);
 248             assert(fault == NoFault || read == NULL);
 249             state = nextState;
 250             nextState = Ready;
 251             if (write)
 252                 walker->port.sendAtomic(write);
 253         } while (read);
 254         state = Ready;
 255         nextState = Waiting;
 256     }
 257     return fault;
 258 }
 259
 260 Fault
 261 Walker::WalkerState::startFunctional(Addr &addr, unsigned &logBytes)
 262 {
 263     Fault fault = NoFault;
 264     assert(!started);
 265     started = true;
 266     setupWalk(addr);
 267
 268     do {
 269         walker->port.sendFunctional(read);
 270         // On a functional access (page table lookup), writes should
 271         // not happen so this pointer is ignored after stepWalk
 272         PacketPtr write = NULL;
 273         fault = stepWalk(write);
 274         assert(fault == NoFault || read == NULL);
 275         state = nextState;
 276         nextState = Ready;
 277     } while (read);
 278     logBytes = entry.logBytes;
 279     addr = entry.paddr << PageShift;
 280
 281     return fault;
 282 }
 283
 284 Fault
 285 Walker::WalkerState::stepWalk(PacketPtr &write)
 286 {
 287     assert(state != Ready && state != Waiting);
 288     Fault fault = NoFault;
 289     write = NULL;
 290     PTESv39 pte = read->getLE<uint64_t>();
 291     Addr nextRead = 0;
 292     bool doWrite = false;
 293     bool doTLBInsert = false;
 294     bool doEndWalk = false;
 295
 296     DPRINTF(PageTableWalker, "Got level%d PTE: %#x\n", level, pte);
 297
 298     // step 2: TODO check PMA and PMP
 299
 300     // step 3:
 301     if (!pte.v || (!pte.r && pte.w)) {
 302         doEndWalk = true;
 303         DPRINTF(PageTableWalker, "PTE invalid, raising PF\n");
 304         fault = pageFault(pte.v);
 305     }
 306     else {
 307         // step 4:
 308         if (pte.r || pte.x) {
 309             // step 5: leaf PTE
 310             doEndWalk = true;
 311             fault = walker->tlb->checkPermissions(status, pmode,
 312                                                   entry.vaddr, mode, pte);
 313
 314             // step 6
 315             if (fault == NoFault) {
 316                 if (level >= 1 && pte.ppn0 != 0) {
 317                     DPRINTF(PageTableWalker,
 318                             "PTE has misaligned PPN, raising PF\n");
 319                     fault = pageFault(true);
 320                 }
 321                 else if (level == 2 && pte.ppn1 != 0) {
 322                     DPRINTF(PageTableWalker,
 323                             "PTE has misaligned PPN, raising PF\n");
 324                     fault = pageFault(true);
 325                 }
 326             }
 327
 328             if (fault == NoFault) {
 329                 // step 7
 330                 if (!pte.a) {
 331                     pte.a = 1;
 332                     doWrite = true;
 333                 }
 334                 if (!pte.d && mode == TLB::Write) {
 335                     pte.d = 1;
 336                     doWrite = true;
 337                 }
 338                 // TODO check if this violates a PMA or PMP
 339
 340                 // step 8
 341                 entry.logBytes = PageShift + (level * LEVEL_BITS);
 342                 entry.paddr = pte.ppn;
 343                 entry.vaddr &= ~((1 << entry.logBytes) - 1);
 344                 entry.pte = pte;
 345                 // put it non-writable into the TLB to detect writes and redo
 346                 // the page table walk in order to update the dirty flag.
 347                 if (!pte.d && mode != TLB::Write)
 348                     entry.pte.w = 0;
 349                 doTLBInsert = true;
 350             }
 351         }
 352         else {
 353             level--;
 354             if (level < 0) {
 355                 DPRINTF(PageTableWalker, "No leaf PTE found, raising PF\n");
 356                 doEndWalk = true;
 357                 fault = pageFault(true);
 358             }
 359             else {
 360                 Addr shift = (PageShift + LEVEL_BITS * level);
 361                 Addr idx = (entry.vaddr >> shift) & LEVEL_MASK;
 362                 nextRead = (pte.ppn << PageShift) + (idx * sizeof(pte));
 363                 nextState = Translate;
 364             }
 365         }
 366     }
 367
 368     PacketPtr oldRead = read;
 369     Request::Flags flags = oldRead->req->getFlags();
 370
 371     if (doEndWalk) {
 372         // If we need to write, adjust the read packet to write the modified
 373         // value back to memory.
 374         if (!functional && doWrite) {
 375             DPRINTF(PageTableWalker, "Writing level%d PTE to %#x: %#x\n",
 376                 level, oldRead->getAddr(), pte);
 377             write = oldRead;
 378             write->setLE<uint64_t>(pte);
 379             write->cmd = MemCmd::WriteReq;
 380             read = NULL;
 381         } else {
 382             write = NULL;
 383         }
 384
 385         if (doTLBInsert) {
 386             if (!functional)
 387                 walker->tlb->insert(entry.vaddr, entry);
 388             else {
 389                 DPRINTF(PageTableWalker, "Translated %#x -> %#x\n",
 390                         entry.vaddr, entry.paddr << PageShift |
 391                         (entry.vaddr & mask(entry.logBytes)));
 392             }
 393         }
 394         endWalk();
 395     }
 396     else {
 397         //If we didn't return, we're setting up another read.
 398         RequestPtr request = std::make_shared<Request>(
 399             nextRead, oldRead->getSize(), flags, walker->requestorId);
 400         read = new Packet(request, MemCmd::ReadReq);
 401         read->allocate();
 402
 403         DPRINTF(PageTableWalker,
 404                 "Loading level%d PTE from %#x\n", level, nextRead);
 405     }
 406
 407     return fault;
 408 }
 409
 410 void
 411 Walker::WalkerState::endWalk()
 412 {
 413     nextState = Ready;
 414     delete read;
 415     read = NULL;
 416 }
 417
 418 void
 419 Walker::WalkerState::setupWalk(Addr vaddr)
 420 {
 421     vaddr &= (static_cast<Addr>(1) << VADDR_BITS) - 1;
 422
 423     Addr shift = PageShift + LEVEL_BITS * 2;
 424     Addr idx = (vaddr >> shift) & LEVEL_MASK;
 425     Addr topAddr = (satp.ppn << PageShift) + (idx * sizeof(PTESv39));
 426     level = 2;
 427
 428     DPRINTF(PageTableWalker, "Performing table walk for address %#x\n", vaddr);
 429     DPRINTF(PageTableWalker, "Loading level%d PTE from %#x\n", level, topAddr);
 430
 431     state = Translate;
 432     nextState = Ready;
 433     entry.vaddr = vaddr;
 434     entry.asid = satp.asid;
 435
 436     Request::Flags flags = Request::PHYSICAL;
 437     RequestPtr request = std::make_shared<Request>(
 438         topAddr, sizeof(PTESv39), flags, walker->requestorId);
 439
 440     read = new Packet(request, MemCmd::ReadReq);
 441     read->allocate();
 442 }
 443
 444 bool
 445 Walker::WalkerState::recvPacket(PacketPtr pkt)
 446 {
 447     assert(pkt->isResponse());
 448     assert(inflight);
 449     assert(state == Waiting);
 450     inflight--;
 451     if (squashed) {
 452         // if were were squashed, return true once inflight is zero and
 453         // this WalkerState will be freed there.
 454         return (inflight == 0);
 455     }
 456     if (pkt->isRead()) {
 457         // should not have a pending read it we also had one outstanding
 458         assert(!read);
 459
 460         // @todo someone should pay for this
 461         pkt->headerDelay = pkt->payloadDelay = 0;
 462
 463         state = nextState;
 464         nextState = Ready;
 465         PacketPtr write = NULL;
 466         read = pkt;
 467         timingFault = stepWalk(write);
 468         state = Waiting;
 469         assert(timingFault == NoFault || read == NULL);
 470         if (write) {
 471             writes.push_back(write);
 472         }
 473         sendPackets();
 474     } else {
 475         sendPackets();
 476     }
 477     if (inflight == 0 && read == NULL && writes.size() == 0) {
 478         state = Ready;
 479         nextState = Waiting;
 480         if (timingFault == NoFault) {
 481             /*
 482              * Finish the translation. Now that we know the right entry is
 483              * in the TLB, this should work with no memory accesses.
 484              * There could be new faults unrelated to the table walk like
 485              * permissions violations, so we'll need the return value as
 486              * well.
 487              */
 488             Addr vaddr = req->getVaddr();
 489             vaddr &= (static_cast<Addr>(1) << VADDR_BITS) - 1;
 490             Addr paddr = walker->tlb->translateWithTLB(vaddr, satp.asid, mode);
 491             req->setPaddr(paddr);
 492             // Let the CPU continue.
 493             translation->finish(NoFault, req, tc, mode);
 494         } else {
 495             // There was a fault during the walk. Let the CPU know.
 496             translation->finish(timingFault, req, tc, mode);
 497         }
 498         return true;
 499     }
 500
 501     return false;
 502 }
 503
 504 void
 505 Walker::WalkerState::sendPackets()
 506 {
 507     //If we're already waiting for the port to become available, just return.
 508     if (retrying)
 509         return;
 510
 511     //Reads always have priority
 512     if (read) {
 513         PacketPtr pkt = read;
 514         read = NULL;
 515         inflight++;
 516         if (!walker->sendTiming(this, pkt)) {
 517             retrying = true;
 518             read = pkt;
 519             inflight--;
 520             return;
 521         }
 522     }
 523     //Send off as many of the writes as we can.
 524     while (writes.size()) {
 525         PacketPtr write = writes.back();
 526         writes.pop_back();
 527         inflight++;
 528         if (!walker->sendTiming(this, write)) {
 529             retrying = true;
 530             writes.push_back(write);
 531             inflight--;
 532             return;
 533         }
 534     }
 535 }
 536
 537 unsigned
 538 Walker::WalkerState::numInflight() const
 539 {
 540     return inflight;
 541 }
 542
 543 bool
 544 Walker::WalkerState::isRetrying()
 545 {
 546     return retrying;
 547 }
 548
 549 bool
 550 Walker::WalkerState::isTiming()
 551 {
 552     return timing;
 553 }
 554
 555 bool
 556 Walker::WalkerState::wasStarted()
 557 {
 558     return started;
 559 }
 560
 561 void
 562 Walker::WalkerState::squash()
 563 {
 564     squashed = true;
 565 }
 566
 567 void
 568 Walker::WalkerState::retry()
 569 {
 570     retrying = false;
 571     sendPackets();
 572 }
 573
 574 Fault
 575 Walker::WalkerState::pageFault(bool present)
 576 {
 577     DPRINTF(PageTableWalker, "Raising page fault.\n");
 578     return walker->tlb->createPagefault(entry.vaddr, mode);
 579 }
 580
 581 } /* end namespace RiscvISA */
 582
 583 RiscvISA::Walker *
 584 RiscvPagetableWalkerParams::create()
 585 {
 586     return new RiscvISA::Walker(this);
 587 }