From 32fb22e361d45e0a6cd8085d69bb01e8ffe33303 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Fri, 24 Jan 2020 00:47:43 -0800 Subject: [PATCH] kern: finish implementing assembly init routines --- libraries/config/common.mk | 8 +- .../libmesosphere/include/mesosphere.hpp | 1 + .../arch/arm64/init/kern_k_init_arguments.hpp | 34 ++++++ .../arm64/init/kern_k_init_page_table.hpp | 27 +++++ .../mesosphere/arch/arm64/kern_cpu.hpp | 13 ++ .../init/kern_init_arguments_select.hpp | 29 +++++ mesosphere/kernel/kernel.ld | 2 +- .../source/arch/arm64/init/kern_init_core.cpp | 46 +++++++ .../source/arch/arm64/{ => init}/start.s | 113 +++++++++++++++++- .../kernel_ldr/source/arch/arm64/start.s | 6 +- .../kernel_ldr/source/kern_init_loader.cpp | 31 +---- 11 files changed, 270 insertions(+), 40 deletions(-) create mode 100644 libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp create mode 100644 mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp rename mesosphere/kernel/source/arch/arm64/{ => init}/start.s (72%) diff --git a/libraries/config/common.mk b/libraries/config/common.mk index 937e10d2c..db2074599 100644 --- a/libraries/config/common.mk +++ b/libraries/config/common.mk @@ -76,14 +76,14 @@ TARGET := $(notdir $(CURDIR)) BUILD := build DATA := data INCLUDES := include -SOURCES ?= $(foreach d,$(filter-out source/arch source/board,$(wildcard source)),$(call DIR_WILDCARD,$d) $d) +SOURCES ?= source $(foreach d,$(filter-out source/arch source/board source,$(wildcard source)),$(call DIR_WILDCARD,$d) $d) ifneq ($(strip $(wildcard source/$(ATMOSPHERE_ARCH_DIR)/.*)),) -SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_ARCH_DIR)) +SOURCES += source/$(ATMOSPHERE_ARCH_DIR) $(call DIR_WILDCARD,source/$(ATMOSPHERE_ARCH_DIR)) endif ifneq ($(strip $(wildcard source/$(ATMOSPHERE_BOARD_DIR)/.*)),) -SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_BOARD_DIR)) +SOURCES += source/$(ATMOSPHERE_BOARD_DIR $(call DIR_WILDCARD,source/$(ATMOSPHERE_BOARD_DIR)) endif ifneq ($(strip $(wildcard source/$(ATMOSPHERE_OS_DIR)/.*)),) -SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_OS_DIR)) +SOURCES += source/$(ATMOSPHERE_OS_DIR $(call DIR_WILDCARD,source/$(ATMOSPHERE_OS_DIR)) endif diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp index 135b6b5a6..3371f426f 100644 --- a/libraries/libmesosphere/include/mesosphere.hpp +++ b/libraries/libmesosphere/include/mesosphere.hpp @@ -32,6 +32,7 @@ #include "mesosphere/init/kern_init_elf.hpp" #include "mesosphere/init/kern_init_layout.hpp" #include "mesosphere/init/kern_init_page_table_select.hpp" +#include "mesosphere/init/kern_init_arguments_select.hpp" /* Core functionality. */ #include "mesosphere/kern_select_interrupts.hpp" diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp new file mode 100644 index 000000000..cdddbc135 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once + +namespace ams::kern::init { + + struct KInitArguments { + u64 ttbr0; + u64 ttbr1; + u64 tcr; + u64 mair; + u64 cpuactlr; + u64 cpuectlr; + u64 sctlr; + u64 sp; + u64 entrypoint; + u64 argument; + u64 setup_function; + }; + +} \ No newline at end of file diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp index 96d7296b5..58a0c153c 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp @@ -495,4 +495,31 @@ namespace ams::kern::init { }; + class KInitialPageAllocator : public KInitialPageTable::IPageAllocator { + private: + uintptr_t next_address; + public: + constexpr ALWAYS_INLINE KInitialPageAllocator() : next_address(Null) { /* ... */ } + + ALWAYS_INLINE void Initialize(uintptr_t address) { + this->next_address = address; + } + + ALWAYS_INLINE uintptr_t GetFinalState() { + const uintptr_t final_address = this->next_address; + this->next_address = Null; + return final_address; + } + public: + virtual KPhysicalAddress Allocate() override { + MESOSPHERE_ABORT_UNLESS(this->next_address != Null); + const uintptr_t allocated = this->next_address; + this->next_address += PageSize; + std::memset(reinterpret_cast(allocated), 0, PageSize); + return allocated; + } + + /* No need to override free. The default does nothing, and so would we. */ + }; + } diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp index 1e1610c95..86412c3a4 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp @@ -19,6 +19,19 @@ namespace ams::kern::arm64::cpu { +#if defined(ATMOSPHERE_CPU_ARM_CORTEX_A57) || defined(ATMOSPHERE_CPU_ARM_CORTEX_A53) + constexpr inline size_t InstructionCacheLineSize = 0x40; + constexpr inline size_t DataCacheLineSize = 0x40; +#else + #error "Unknown CPU for cache line sizes" +#endif + +#if defined(ATMOSPHERE_BOARD_NINTENDO_SWITCH) + static constexpr size_t NumCores = 4; +#else + #error "Unknown Board for cpu::NumCores" +#endif + /* Helpers for managing memory state. */ ALWAYS_INLINE void DataSynchronizationBarrier() { __asm__ __volatile__("dsb sy" ::: "memory"); diff --git a/libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp b/libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp new file mode 100644 index 000000000..6c124528d --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +#ifdef ATMOSPHERE_ARCH_ARM64 + #include "../arch/arm64/init/kern_k_init_arguments.hpp" +#else + #error "Unknown architecture for KInitArguments" +#endif + +namespace ams::kern::init { + + KPhysicalAddress GetInitArgumentsAddress(s32 core_id); + +} diff --git a/mesosphere/kernel/kernel.ld b/mesosphere/kernel/kernel.ld index 08648015b..2d6497b27 100644 --- a/mesosphere/kernel/kernel.ld +++ b/mesosphere/kernel/kernel.ld @@ -18,7 +18,7 @@ SECTIONS .crt0 : { - KEEP (*(.crt0)) + KEEP (*(.crt0 .crt0.*)) . = ALIGN(8); } :code diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp new file mode 100644 index 000000000..9f8bdf691 --- /dev/null +++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern::init { + + namespace { + + /* Global Allocator. */ + KInitialPageAllocator g_initial_page_allocator; + + /* Global initial arguments array. */ + KInitArguments g_init_arguments[cpu::NumCores]; + + } + + void InitializeCore(uintptr_t arg0, uintptr_t initial_page_allocator_state) { + /* TODO */ + } + + KPhysicalAddress GetInitArgumentsAddress(s32 core) { + return KPhysicalAddress(std::addressof(g_init_arguments[core])); + } + + void InitializeDebugRegisters() { + /* TODO */ + } + + void InitializeExceptionVectors() { + /* TODO */ + } + +} \ No newline at end of file diff --git a/mesosphere/kernel/source/arch/arm64/start.s b/mesosphere/kernel/source/arch/arm64/init/start.s similarity index 72% rename from mesosphere/kernel/source/arch/arm64/start.s rename to mesosphere/kernel/source/arch/arm64/init/start.s index c1a22ce98..e769e7c14 100644 --- a/mesosphere/kernel/source/arch/arm64/start.s +++ b/mesosphere/kernel/source/arch/arm64/init/start.s @@ -94,9 +94,116 @@ core0_el1: add x3, x0, x3 blr x3 - /* TODO: Finish post-kernelldr init code. */ -1: - b 1b + /* At this point kernelldr has been invoked, and we are relocated at a random virtual address. */ + /* Next thing to do is to set up our memory management and slabheaps -- all the other core initialization. */ + /* Call ams::kern::init::InitializeCore(uintptr_t, uintptr_t) */ + mov x1, x0 /* Kernelldr returns a KInitialPageAllocator state for the kernel to re-use. */ + mov x0, xzr /* Official kernel always passes zero, when this is non-zero the address is mapped. */ + bl _ZN3ams4kern4init14InitializeCoreEmm + + /* Get the init arguments for core 0. */ + mov x0, xzr + bl _ZN3ams4kern4init23GetInitArgumentsAddressEi + + bl _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE + +/* ams::kern::init::StartOtherCore(const ams::kern::init::KInitArguments *) */ +.section .crt0.text._ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE, "ax", %progbits +.global _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE +.type _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE, %function +_ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE: + /* Preserve the KInitArguments pointer in a register. */ + mov x20, x0 + + /* Check our current EL. We want to be executing out of EL1. */ + /* If we're in EL2, we'll need to deprivilege ourselves. */ + mrs x1, currentel + cmp x1, #0x4 + b.eq othercore_el1 + cmp x1, #0x8 + b.eq othercore_el2 +othercore_el3: + b othercore_el3 +othercore_el2: + bl _ZN3ams4kern4init16JumpFromEL2ToEL1Ev +othercore_el1: + bl _ZN3ams4kern4init19DisableMmuAndCachesEv + + /* Setup system registers using values from our KInitArguments. */ + ldr x1, [x20, #0x00] + msr ttbr0_el1, x1 + ldr x1, [x20, #0x08] + msr ttbr1_el1, x1 + ldr x1, [x20, #0x10] + msr tcr_el1, x1 + ldr x1, [x20, #0x18] + msr mair_el1, x1 + + /* Perform cpu-specific setup. */ + mrs x1, midr_el1 + ubfx x2, x1, #0x18, #0x8 /* Extract implementer bits. */ + cmp x2, #0x41 /* Implementer::ArmLimited */ + b.ne othercore_cpu_specific_setup_end + ubfx x2, x1, #0x4, #0xC /* Extract primary part number. */ + cmp x2, #0xD07 /* PrimaryPartNumber::CortexA57 */ + b.eq othercore_cpu_specific_setup_cortex_a57 + cmp x2, #0xD03 /* PrimaryPartNumber::CortexA53 */ + b.eq othercore_cpu_specific_setup_cortex_a53 + b othercore_cpu_specific_setup_end +othercore_cpu_specific_setup_cortex_a57: +othercore_cpu_specific_setup_cortex_a53: + ldr x1, [x20, #0x20] + msr cpuactlr_el1, x1 + ldr x1, [x20, #0x28] + msr cpuectlr_el1, x1 + +othercore_cpu_specific_setup_end: + /* Ensure instruction consistency. */ + dsb sy + isb + + /* Set sctlr_el1 and ensure instruction consistency. */ + ldr x1, [x20, #0x30] + msr sctlr_el1, x1 + + dsb sy + isb + + /* Jump to the virtual address equivalent to ams::kern::init::InvokeEntrypoint */ + ldr x1, [x20, #0x50] + adr x2, _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE + sub x1, x1, x2 + adr x2, _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE + add x1, x1, x2 + mov x0, x20 + br x1 + +/* ams::kern::init::InvokeEntrypoint(const ams::kern::init::KInitArguments *) */ +.section .crt0.text._ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE, "ax", %progbits +.global _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE +.type _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE, %function +_ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE: + /* Preserve the KInitArguments pointer in a register. */ + mov x20, x0 + + /* Clear CPACR_EL1. This will prevent classes of traps (SVE, etc). */ + msr cpacr_el1, xzr + isb + + /* Setup the stack pointer. */ + ldr x1, [x20, #0x38] + mov sp, x1 + + /* Ensure that system debug registers are setup. */ + bl _ZN3ams4kern4init24InitializeDebugRegistersEv + + /* Ensure that the exception vectors are setup. */ + bl _ZN3ams4kern4init26InitializeExceptionVectorsEv + + /* Jump to the entrypoint. */ + ldr x1, [x20, #0x40] + ldr x0, [x20, #0x48] + br x1 /* ams::kern::init::JumpFromEL2ToEL1() */ diff --git a/mesosphere/kernel_ldr/source/arch/arm64/start.s b/mesosphere/kernel_ldr/source/arch/arm64/start.s index eaf209211..7d2eacadc 100644 --- a/mesosphere/kernel_ldr/source/arch/arm64/start.s +++ b/mesosphere/kernel_ldr/source/arch/arm64/start.s @@ -75,10 +75,10 @@ _start: bl _ZN3ams4kern4init6loader4MainEmPNS1_12KernelLayoutEm str x0, [sp, #0x00] - /* Call ams::kern::init::loader::Finalize() */ - bl _ZN3ams4kern4init6loader8FinalizeEv + /* Call ams::kern::init::loader::GetFinalPageAllocatorState() */ + bl _ZN3ams4kern4init6loader26GetFinalPageAllocatorStateEv - /* X0 is now the next address for the page allocator. */ + /* X0 is now the saved state for the page allocator. */ /* We will return this to the kernel. */ /* Return to the newly-relocated kernel. */ diff --git a/mesosphere/kernel_ldr/source/kern_init_loader.cpp b/mesosphere/kernel_ldr/source/kern_init_loader.cpp index c00477270..ca9b0e272 100644 --- a/mesosphere/kernel_ldr/source/kern_init_loader.cpp +++ b/mesosphere/kernel_ldr/source/kern_init_loader.cpp @@ -34,33 +34,6 @@ namespace ams::kern::init::loader { constexpr size_t InitialPageTableRegionSize = 0x200000; - class KInitialPageAllocator : public KInitialPageTable::IPageAllocator { - private: - uintptr_t next_address; - public: - constexpr ALWAYS_INLINE KInitialPageAllocator() : next_address(Null) { /* ... */ } - - ALWAYS_INLINE void Initialize(uintptr_t address) { - this->next_address = address; - } - - ALWAYS_INLINE uintptr_t Finalize() { - const uintptr_t final_address = this->next_address; - this->next_address = Null; - return final_address; - } - public: - virtual KPhysicalAddress Allocate() override { - MESOSPHERE_ABORT_UNLESS(this->next_address != Null); - const uintptr_t allocated = this->next_address; - this->next_address += PageSize; - std::memset(reinterpret_cast(allocated), 0, PageSize); - return allocated; - } - - /* No need to override free. The default does nothing, and so would we. */ - }; - /* Global Allocator. */ KInitialPageAllocator g_initial_page_allocator; @@ -335,8 +308,8 @@ namespace ams::kern::init::loader { return GetInteger(virtual_base_address) - base_address; } - uintptr_t Finalize() { - return g_initial_page_allocator.Finalize(); + uintptr_t GetFinalPageAllocatorState() { + return g_initial_page_allocator.GetFinalState(); } } \ No newline at end of file