From 6ecf110b06d7333aae13bcc16216cc127fdda7f0 Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Mon, 21 Sep 2020 17:11:59 +0100 Subject: [PATCH] arch-arm: inform bootloader of kernel position with a register Before the commit, the bootloader had a hardcoded entry point that it would jump to. However, the Linux kernel arm64 v5.8 forced us to change the kernel entry point because the required memory alignment has changed at: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/ commit/?h=v5.8&id=cfa7ede20f133cc81cef01dc3a516dda3a9721ee Therefore the only way to have a single bootloader that boots both pre-v5.8 and post-v5.8 kernels is to pass that information from gem5 to the bootloader, which we do in this patch via registers. This approach was already used by the 32-bit bootloader, which passed that value via r3, and we try to use the same register x3 in 64-bit. Since we are now passing this information, the this patch also removes the hardcoding of DTB and cpu-release-addr, and also passes those values via registers. We store the cpu-release-addr in x5 as that value appears to have a function similar to flags_addr, which is used only in 32-bit arm and gets stored in r5. This commit renames atags_addr to dtb_addr, since both are mutually exclusive, and serve a similar purpose, DTB being the newer recommended approach. Similarly, flags_addr is renamed to cpu_release_addr, and it is moved from ArmSystem into ArmFsWorkload, since it is not an intrinsic system property, and should be together with dtb_addr instead. Before this commit, flags_addr was being set from FSConfig.py and configs/example/arm/devices.py to self.realview.realview_io.pio_addr + 0x30. This commit moves that logic into RealView.py instead, and sets the flags address 8 bytes before the start of the DTB address. JIRA: https://gem5.atlassian.net/browse/GEM5-787 Change-Id: If70bea9690be04b84e6040e256a9b03e46710e10 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/35076 Reviewed-by: Andreas Sandberg Maintainer: Andreas Sandberg Tested-by: kokoro --- configs/common/FSConfig.py | 4 +--- configs/example/arm/devices.py | 1 - configs/example/arm/workloads.py | 4 ++-- src/arch/arm/ArmFsWorkload.py | 6 ++--- src/arch/arm/ArmSystem.py | 1 - src/arch/arm/freebsd/fs_workload.cc | 6 ++--- src/arch/arm/fs_workload.cc | 14 +++++------ src/arch/arm/linux/fs_workload.cc | 31 +++++++++++++++--------- src/dev/arm/RealView.py | 10 +++++--- system/arm/bootloader/arm64/boot.S | 35 +++++++++++++++++++++++----- system/arm/bootloader/arm64/makefile | 6 +---- 11 files changed, 73 insertions(+), 45 deletions(-) diff --git a/configs/common/FSConfig.py b/configs/common/FSConfig.py index 710ee4d49..6fd39a5aa 100644 --- a/configs/common/FSConfig.py +++ b/configs/common/FSConfig.py @@ -250,7 +250,7 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None, if bare_metal: # EOT character on UART will end the simulation self.realview.uart[0].end_on_eot = True - self.workload = ArmFsWorkload(atags_addr=0) + self.workload = ArmFsWorkload(dtb_addr=0) else: workload = ArmFsLinux() @@ -269,8 +269,6 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None, if hasattr(self.realview.gic, 'cpu_addr'): self.gic_cpu_addr = self.realview.gic.cpu_addr - self.flags_addr = self.realview.realview_io.pio_addr + 0x30 - # This check is for users who have previously put 'android' in # the disk image filename to tell the config scripts to # prepare the kernel with android-specific boot options. That diff --git a/configs/example/arm/devices.py b/configs/example/arm/devices.py index 971782d76..e3cee1e72 100644 --- a/configs/example/arm/devices.py +++ b/configs/example/arm/devices.py @@ -308,7 +308,6 @@ def simpleSystem(BaseSystem, caches, mem_size, platform=None, **kwargs): if hasattr(self.realview.gic, 'cpu_addr'): self.gic_cpu_addr = self.realview.gic.cpu_addr - self.flags_addr = self.realview.realview_io.pio_addr + 0x30 self.membus = MemBus() diff --git a/configs/example/arm/workloads.py b/configs/example/arm/workloads.py index 6952a4a80..ce48cdd7c 100644 --- a/configs/example/arm/workloads.py +++ b/configs/example/arm/workloads.py @@ -47,7 +47,7 @@ from common.SysPaths import binary, disk class ArmBaremetal(ArmFsWorkload): """ Baremetal workload """ - atags_addr = 0 + dtb_addr = 0 def __init__(self, obj, system, **kwargs): super(ArmBaremetal, self).__init__(**kwargs) @@ -72,7 +72,7 @@ class ArmTrustedFirmware(ArmFsWorkload): https://github.com/ARM-software/arm-trusted-firmware """ - atags_addr = 0 + dtb_addr = 0 def __init__(self, obj, system, **kwargs): super(ArmTrustedFirmware, self).__init__(**kwargs) diff --git a/src/arch/arm/ArmFsWorkload.py b/src/arch/arm/ArmFsWorkload.py index bc27c6d79..b57b1f0a2 100644 --- a/src/arch/arm/ArmFsWorkload.py +++ b/src/arch/arm/ArmFsWorkload.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009, 2012-2013, 2015-2019 ARM Limited +# Copyright (c) 2009, 2012-2013, 2015-2020 ARM Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -57,11 +57,11 @@ class ArmFsWorkload(KernelWorkload): dtb_filename = Param.String("", "File that contains the Device Tree Blob. Don't use DTB if empty.") + dtb_addr = Param.Addr(0, "DTB or ATAGS address") + cpu_release_addr = Param.Addr(0, "cpu-release-addr property") machine_type = Param.ArmMachineType('DTOnly', "Machine id from http://www.arm.linux.org.uk/developer/machines/") - atags_addr = Param.Addr("Address where default atags structure should " \ - "be written") early_kernel_symbols = Param.Bool(False, "enable early kernel symbol tables before MMU") enable_context_switch_stats_dump = Param.Bool(False, diff --git a/src/arch/arm/ArmSystem.py b/src/arch/arm/ArmSystem.py index 13f0c2dd2..7ab4b6ec4 100644 --- a/src/arch/arm/ArmSystem.py +++ b/src/arch/arm/ArmSystem.py @@ -48,7 +48,6 @@ class ArmSystem(System): cxx_header = "arch/arm/system.hh" multi_proc = Param.Bool(True, "Multiprocessor system?") gic_cpu_addr = Param.Addr(0, "Addres of the GIC CPU interface") - flags_addr = Param.Addr(0, "Address of the flags register for MP booting") have_security = Param.Bool(False, "True if Security Extensions are implemented") have_virtualization = Param.Bool(False, diff --git a/src/arch/arm/freebsd/fs_workload.cc b/src/arch/arm/freebsd/fs_workload.cc index 241b40e0c..8eb3c70e1 100644 --- a/src/arch/arm/freebsd/fs_workload.cc +++ b/src/arch/arm/freebsd/fs_workload.cc @@ -95,7 +95,7 @@ FsFreebsd::initState() // Kernel supports flattened device tree and dtb file specified. // Using Device Tree Blob to describe system configuration. inform("Loading DTB file: %s at address %#x\n", params().dtb_filename, - params().atags_addr + _loadAddrOffset); + params().dtb_addr + _loadAddrOffset); auto *dtb_file = new ::Loader::DtbFile(params().dtb_filename); @@ -108,7 +108,7 @@ FsFreebsd::initState() bootReleaseAddr = ra & ~ULL(0x7F); dtb_file->buildImage(). - offset(params().atags_addr + _loadAddrOffset). + offset(params().dtb_addr + _loadAddrOffset). write(system->physProxy); delete dtb_file; @@ -116,7 +116,7 @@ FsFreebsd::initState() for (auto *tc: system->threads) { tc->setIntReg(0, 0); tc->setIntReg(1, params().machine_type); - tc->setIntReg(2, params().atags_addr + _loadAddrOffset); + tc->setIntReg(2, params().dtb_addr + _loadAddrOffset); } } diff --git a/src/arch/arm/fs_workload.cc b/src/arch/arm/fs_workload.cc index 84d3a0d1e..b80181722 100644 --- a/src/arch/arm/fs_workload.cc +++ b/src/arch/arm/fs_workload.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2012-2013, 2015,2017-2019 ARM Limited + * Copyright (c) 2010, 2012-2013, 2015,2017-2020 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -117,21 +117,21 @@ FsWorkload::initState() inform("Using bootloader at address %#x", bootldr->entryPoint()); - // Put the address of the boot loader into r7 so we know + // The address of the boot loader so we know // where to branch to after the reset fault // All other values needed by the boot loader to know what to do - fatal_if(!arm_sys->params().flags_addr, - "flags_addr must be set with bootloader"); + fatal_if(!params().cpu_release_addr, + "cpu_release_addr must be set with bootloader"); fatal_if(!arm_sys->params().gic_cpu_addr && is_gic_v2, "gic_cpu_addr must be set with bootloader"); for (auto *tc: arm_sys->threads) { - if (!arm_sys->highestELIs64()) - tc->setIntReg(3, kernelEntry); + tc->setIntReg(3, kernelEntry); if (is_gic_v2) tc->setIntReg(4, arm_sys->params().gic_cpu_addr); - tc->setIntReg(5, arm_sys->params().flags_addr); + if (getArch() == Loader::Arm) + tc->setIntReg(5, params().cpu_release_addr); } inform("Using kernel entry physical address at %#x\n", kernelEntry); } else { diff --git a/src/arch/arm/linux/fs_workload.cc b/src/arch/arm/linux/fs_workload.cc index 54d5c863e..f05fd6faa 100644 --- a/src/arch/arm/linux/fs_workload.cc +++ b/src/arch/arm/linux/fs_workload.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2013, 2016 ARM Limited + * Copyright (c) 2010-2013, 2016, 2020 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -92,7 +92,7 @@ FsLinux::initState() // Kernel supports flattened device tree and dtb file specified. // Using Device Tree Blob to describe system configuration. inform("Loading DTB file: %s at address %#x\n", params().dtb_filename, - params().atags_addr + _loadAddrOffset); + params().dtb_addr); auto *dtb_file = new ::Loader::DtbFile(params().dtb_filename); @@ -102,9 +102,8 @@ FsLinux::initState() params().dtb_filename); } - dtb_file->buildImage(). - offset(params().atags_addr + _loadAddrOffset). - write(system->physProxy); + dtb_file->buildImage().offset(params().dtb_addr) + .write(system->physProxy); delete dtb_file; } else { // Using ATAGS @@ -152,17 +151,27 @@ FsLinux::initState() DPRINTF(Loader, "Boot atags was %d bytes in total\n", size << 2); DDUMP(Loader, boot_data, size << 2); - system->physProxy.writeBlob(params().atags_addr + _loadAddrOffset, + system->physProxy.writeBlob(params().dtb_addr + _loadAddrOffset, boot_data, size << 2); delete[] boot_data; } - // Kernel boot requirements to set up r0, r1 and r2 in ARMv7 - for (auto *tc: system->threads) { - tc->setIntReg(0, 0); - tc->setIntReg(1, params().machine_type); - tc->setIntReg(2, params().atags_addr + _loadAddrOffset); + if (getArch() == Loader::Arm64) { + // We inform the bootloader of the kernel entry point. This was added + // originally done because the entry offset changed in kernel v5.8. + // Previously the bootloader just used a hardcoded address. + for (auto *tc: system->threads) { + tc->setIntReg(0, params().dtb_addr); + tc->setIntReg(5, params().cpu_release_addr); + } + } else { + // Kernel boot requirements to set up r0, r1 and r2 in ARMv7 + for (auto *tc: system->threads) { + tc->setIntReg(0, 0); + tc->setIntReg(1, params().machine_type); + tc->setIntReg(2, params().dtb_addr + _loadAddrOffset); + } } } diff --git a/src/dev/arm/RealView.py b/src/dev/arm/RealView.py index 6f12a8b6b..8fa0edde1 100644 --- a/src/dev/arm/RealView.py +++ b/src/dev/arm/RealView.py @@ -713,10 +713,11 @@ class RealView(Platform): self._attach_mem(self._off_chip_memory(), bus, mem_ports) self._attach_io(self._off_chip_devices(), bus, dma_ports) - def setupBootLoader(self, cur_sys, boot_loader, atags_addr, load_offset): + def setupBootLoader(self, cur_sys, boot_loader, dtb_addr, load_offset): cur_sys.workload.boot_loader = boot_loader - cur_sys.workload.atags_addr = atags_addr cur_sys.workload.load_addr_offset = load_offset + cur_sys.workload.dtb_addr = load_offset + dtb_addr + cur_sys.workload.cpu_release_addr = cur_sys.workload.dtb_addr - 8 def generateDeviceTree(self, state): node = FdtNode("/") # Things in this module need to end up in the root @@ -734,8 +735,11 @@ class RealView(Platform): cpu.append(FdtPropertyStrings('enable-method', 'psci')) else: cpu.append(FdtPropertyStrings("enable-method", "spin-table")) + # The kernel writes the entry addres of secondary CPUs to this + # address before waking up secondary CPUs. + # The gem5 bootloader then makes secondary CPUs jump to it. cpu.append(FdtPropertyWords("cpu-release-addr", \ - state.addrCells(0x8000fff8))) + state.addrCells(system.workload.cpu_release_addr))) class VExpress_EMM(RealView): _mem_regions = [ AddrRange('2GB', size='2GB') ] diff --git a/system/arm/bootloader/arm64/boot.S b/system/arm/bootloader/arm64/boot.S index b3baa71b0..8af177f26 100644 --- a/system/arm/bootloader/arm64/boot.S +++ b/system/arm/bootloader/arm64/boot.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 ARM Limited + * Copyright (c) 2012, 2020 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -40,6 +40,14 @@ .globl _start _start: + /* Save some values initialized by gem5. */ + /* DTB address. */ + mov x21, x0 + /* Kernel entry point. */ + mov x20, x3 + /* cpu-release-addr. */ + mov x22, x5 + /* * EL3 initialisation */ @@ -153,8 +161,19 @@ start_ns: * Secondary CPUs */ 1: wfe - ldr x4, =PHYS_OFFSET + 0xfff8 - ldr x4, [x4] + /* The Linux kernel v5.8 and older writes the entry point address + * of the secondary CPUs to this address, and does a SEV, waking up + * the secondary CPUs. + * + * gem5 informs the kernel the desired address via cpu-release-addr + * of the DTB. + * + * When this is first reached immediately after the bootloader starts, + * the value at that address must be 0, which is the default memory + * value set by gem5 for otherwise uninitialized memory, leading to + * WFE. + */ + ldr x4, [x22] cbz x4, 1b br x4 // branch to the given address @@ -180,9 +199,13 @@ start_ns: /* * Primary CPU */ - ldr x0, =PHYS_OFFSET + 0x8000000 // device tree blob - ldr x6, =PHYS_OFFSET + 0x80000 // kernel start address - br x6 + // The kernel boot protocol specifies that the DTB address is placed + // in x0. + // https://github.com/torvalds/linux/blob/v5.7/Documentation/arm64/ + // booting.rst#4-call-the-kernel-image + mov x0, x21 + // Jump into the kernel entry point. + br x20 .ltorg diff --git a/system/arm/bootloader/arm64/makefile b/system/arm/bootloader/arm64/makefile index 2112b6e71..dbf7128ef 100644 --- a/system/arm/bootloader/arm64/makefile +++ b/system/arm/bootloader/arm64/makefile @@ -34,11 +34,7 @@ BUILDDIR = . DESTDIR = $(error Please set DESTDIR to wanted installation directory) CFLAGS = -march=armv8-a -CPPFLAGS = -DPHYS_OFFSET=0x80000000 \ - -DUART_BASE=0x1c090000 -DSYSREGS_BASE=0x1c010000 \ - -Dkernel=0x80080000 \ - -Dmbox=0x8000fff8 -Ddtb=0x80000100 - +CPPFLAGS = -DUART_BASE=0x1c090000 -DSYSREGS_BASE=0x1c010000 LDFLAGS = -N -Ttext 0x00000010 -static .PHONY: all clean install mkdir -- 2.30.2