From a72e39d657b65e9ef3a7f0e1dac367603b664cc8 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Wed, 9 Oct 2024 14:04:15 -0700 Subject: [PATCH] kern: allocate all TTBR0 pages during init, use procidx as asid --- .../arch/arm64/kern_k_page_table.hpp | 25 ++++- .../arch/arm64/kern_k_process_page_table.hpp | 8 +- .../arm64/kern_k_supervisor_page_table.hpp | 3 +- .../include/mesosphere/kern_k_process.hpp | 2 +- .../source/arch/arm64/kern_k_page_table.cpp | 103 ++---------------- .../nintendo/nx/kern_k_sleep_manager.cpp | 4 +- .../libmesosphere/source/kern_k_process.cpp | 4 +- mesosphere/kernel/kernel.ld | 2 + .../source/arch/arm64/init/kern_init_core.cpp | 31 ++++++ 9 files changed, 72 insertions(+), 110 deletions(-) diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp index 14aba9913..370211b2b 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp @@ -93,9 +93,13 @@ namespace ams::kern::arch::arm64 { MESOSPHERE_ASSERT(alignment < L1BlockSize); return KPageTable::GetBlockSize(static_cast(KPageTable::GetBlockType(alignment) + 1)); } + public: + /* TODO: How should this size be determined. Does the KProcess slab count need to go in a header as a define? */ + static constexpr size_t NumTtbr0Entries = 81; + private: + static constinit inline const volatile u64 s_ttbr0_entries[NumTtbr0Entries] = {}; private: KPageTableManager *m_manager; - u64 m_ttbr; u8 m_asid; protected: Result OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll); @@ -168,17 +172,28 @@ namespace ams::kern::arch::arm64 { return entry; } public: - constexpr explicit KPageTable(util::ConstantInitializeTag) : KPageTableBase(util::ConstantInitialize), m_manager(), m_ttbr(), m_asid() { /* ... */ } + constexpr explicit KPageTable(util::ConstantInitializeTag) : KPageTableBase(util::ConstantInitialize), m_manager(), m_asid() { /* ... */ } explicit KPageTable() { /* ... */ } static NOINLINE void Initialize(s32 core_id); - ALWAYS_INLINE void Activate(u32 proc_id) { - cpu::SwitchProcess(m_ttbr, proc_id); + static const volatile u64 &GetTtbr0Entry(size_t index) { return s_ttbr0_entries[index]; } + + static ALWAYS_INLINE u64 GetKernelTtbr0() { + return s_ttbr0_entries[0]; + } + + static ALWAYS_INLINE void ActivateKernel() { + /* Activate, using asid 0 and process id = 0xFFFFFFFF */ + cpu::SwitchProcess(GetKernelTtbr0(), 0xFFFFFFFF); + } + + static ALWAYS_INLINE void ActivateProcess(size_t proc_idx, u32 proc_id) { + cpu::SwitchProcess(s_ttbr0_entries[proc_idx + 1], proc_id); } NOINLINE Result InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end); - NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit); + NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index); Result Finalize(); private: Result MapL1Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll); diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp index 640b9e5e5..3ad3c96a1 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp @@ -23,13 +23,13 @@ namespace ams::kern::arch::arm64 { private: KPageTable m_page_table; public: - void Activate(u64 id) { + void Activate(size_t process_index, u64 id) { /* Activate the page table with the specified contextidr. */ - m_page_table.Activate(id); + m_page_table.ActivateProcess(process_index, id); } - Result Initialize(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit) { - R_RETURN(m_page_table.InitializeForProcess(flags, from_back, pool, code_address, code_size, system_resource, resource_limit)); + Result Initialize(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index) { + R_RETURN(m_page_table.InitializeForProcess(flags, from_back, pool, code_address, code_size, system_resource, resource_limit, process_index)); } void Finalize() { m_page_table.Finalize(); } diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp index 73d886d84..0c0602289 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp @@ -29,8 +29,7 @@ namespace ams::kern::arch::arm64 { NOINLINE void Initialize(s32 core_id); void Activate() { - /* Activate, using process id = 0xFFFFFFFF */ - m_page_table.Activate(0xFFFFFFFF); + m_page_table.ActivateKernel(); } void ActivateForInit() { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp index 5a37e3a6b..c4ab7f2a5 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -374,7 +374,7 @@ namespace ams::kern { /* Update the current page table. */ if (next_process) { - next_process->GetPageTable().Activate(next_process->GetProcessId()); + next_process->GetPageTable().Activate(next_process->GetSlabIndex(), next_process->GetProcessId()); } else { Kernel::GetKernelPageTable().Activate(); } diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp index 55e4fc52b..c1b0c76b4 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp @@ -85,77 +85,6 @@ namespace ams::kern::arch::arm64 { return (static_cast(asid) << 48) | (static_cast(GetInteger(table))); } - class KPageTableAsidManager { - private: - using WordType = u32; - static constexpr u8 ReservedAsids[] = { 0 }; - static constexpr size_t NumReservedAsids = util::size(ReservedAsids); - static constexpr size_t BitsPerWord = BITSIZEOF(WordType); - static constexpr size_t AsidCount = 0x100; - static constexpr size_t NumWords = AsidCount / BitsPerWord; - static constexpr WordType FullWord = ~WordType(0u); - private: - WordType m_state[NumWords]; - KLightLock m_lock; - u8 m_hint; - private: - constexpr bool TestImpl(u8 asid) const { - return m_state[asid / BitsPerWord] & (1u << (asid % BitsPerWord)); - } - constexpr void ReserveImpl(u8 asid) { - MESOSPHERE_ASSERT(!this->TestImpl(asid)); - m_state[asid / BitsPerWord] |= (1u << (asid % BitsPerWord)); - } - - constexpr void ReleaseImpl(u8 asid) { - MESOSPHERE_ASSERT(this->TestImpl(asid)); - m_state[asid / BitsPerWord] &= ~(1u << (asid % BitsPerWord)); - } - - constexpr u8 FindAvailable() const { - for (size_t i = 0; i < util::size(m_state); i++) { - if (m_state[i] == FullWord) { - continue; - } - const WordType clear_bit = (m_state[i] + 1) ^ (m_state[i]); - return BitsPerWord * i + BitsPerWord - 1 - ClearLeadingZero(clear_bit); - } - if (m_state[util::size(m_state)-1] == FullWord) { - MESOSPHERE_PANIC("Unable to reserve ASID"); - } - __builtin_unreachable(); - } - - static constexpr ALWAYS_INLINE WordType ClearLeadingZero(WordType value) { - return __builtin_clzll(value) - (BITSIZEOF(unsigned long long) - BITSIZEOF(WordType)); - } - public: - constexpr KPageTableAsidManager() : m_state(), m_lock(), m_hint() { - for (size_t i = 0; i < NumReservedAsids; i++) { - this->ReserveImpl(ReservedAsids[i]); - } - } - - u8 Reserve() { - KScopedLightLock lk(m_lock); - - if (this->TestImpl(m_hint)) { - m_hint = this->FindAvailable(); - } - - this->ReserveImpl(m_hint); - - return m_hint++; - } - - void Release(u8 asid) { - KScopedLightLock lk(m_lock); - this->ReleaseImpl(asid); - } - }; - - KPageTableAsidManager g_asid_manager; - } ALWAYS_INLINE void KPageTable::NoteUpdated() const { @@ -184,6 +113,7 @@ namespace ams::kern::arch::arm64 { this->OnKernelTableSinglePageUpdated(virt_addr); } + void KPageTable::Initialize(s32 core_id) { /* Nothing actually needed here. */ MESOSPHERE_UNUSED(core_id); @@ -194,38 +124,29 @@ namespace ams::kern::arch::arm64 { m_asid = 0; m_manager = Kernel::GetSystemSystemResource().GetPageTableManagerPointer(); - /* Allocate a page for ttbr. */ - /* NOTE: It is a postcondition of page table manager allocation that the page is all-zero. */ - const u64 asid_tag = (static_cast(m_asid) << 48ul); - const KVirtualAddress page = m_manager->Allocate(); - MESOSPHERE_ASSERT(page != Null); - m_ttbr = GetInteger(KPageTableBase::GetLinearMappedPhysicalAddress(page)) | asid_tag; - /* Initialize the base page table. */ MESOSPHERE_R_ABORT_UNLESS(KPageTableBase::InitializeForKernel(true, table, start, end)); R_SUCCEED(); } - Result KPageTable::InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit) { - /* Get an ASID */ - m_asid = g_asid_manager.Reserve(); - ON_RESULT_FAILURE { g_asid_manager.Release(m_asid); }; + Result KPageTable::InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index) { + /* Determine our ASID */ + m_asid = process_index + 1; + MESOSPHERE_ABORT_UNLESS(0 < m_asid && m_asid < util::size(s_ttbr0_entries)); /* Set our manager. */ m_manager = system_resource->GetPageTableManagerPointer(); - /* Allocate a new table, and set our ttbr value. */ - const KVirtualAddress new_table = m_manager->Allocate(); - R_UNLESS(new_table != Null, svc::ResultOutOfResource()); - m_ttbr = EncodeTtbr(GetPageTablePhysicalAddress(new_table), m_asid); - ON_RESULT_FAILURE_2 { m_manager->Free(new_table); }; + /* Get the virtual address of our L1 table. */ + const KPhysicalAddress ttbr0_phys = KPhysicalAddress(s_ttbr0_entries[m_asid] & UINT64_C(0xFFFFFFFFFFFE)); + const KVirtualAddress ttbr0_virt = KMemoryLayout::GetLinearVirtualAddress(ttbr0_phys); /* Initialize our base table. */ const size_t as_width = GetAddressSpaceWidth(flags); const KProcessAddress as_start = 0; const KProcessAddress as_end = (1ul << as_width); - R_TRY(KPageTableBase::InitializeForProcess(flags, from_back, pool, GetVoidPointer(new_table), as_start, as_end, code_address, code_size, system_resource, resource_limit)); + R_TRY(KPageTableBase::InitializeForProcess(flags, from_back, pool, GetVoidPointer(ttbr0_virt), as_start, as_end, code_address, code_size, system_resource, resource_limit)); /* Note that we've updated the table (since we created it). */ this->NoteUpdated(); @@ -329,20 +250,16 @@ namespace ams::kern::arch::arm64 { } } - /* Free the L1 table. */ + /* Clear the L1 table. */ { const KVirtualAddress l1_table = reinterpret_cast(impl.Finalize()); ClearPageTable(l1_table); - this->GetPageTableManager().Free(l1_table); } /* Perform inherited finalization. */ KPageTableBase::Finalize(); } - /* Release our asid. */ - g_asid_manager.Release(m_asid); - R_SUCCEED(); } diff --git a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp index bb782d390..5e5983c86 100644 --- a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp +++ b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp @@ -31,7 +31,6 @@ namespace ams::kern::board::nintendo::nx { /* Struct representing registers saved on wake/sleep. */ class SavedSystemRegisters { private: - u64 ttbr0_el1; u64 elr_el1; u64 sp_el0; u64 spsr_el1; @@ -90,7 +89,6 @@ namespace ams::kern::board::nintendo::nx { void SavedSystemRegisters::Save() { /* Save system registers. */ - this->ttbr0_el1 = cpu::GetTtbr0El1(); this->tpidr_el0 = cpu::GetTpidrEl0(); this->elr_el1 = cpu::GetElrEl1(); this->sp_el0 = cpu::GetSpEl0(); @@ -405,7 +403,7 @@ namespace ams::kern::board::nintendo::nx { cpu::EnsureInstructionConsistency(); /* Restore system registers. */ - cpu::SetTtbr0El1 (this->ttbr0_el1); + cpu::SetTtbr0El1 (KPageTable::GetKernelTtbr0()); cpu::SetTpidrEl0 (this->tpidr_el0); cpu::SetElrEl1 (this->elr_el1); cpu::SetSpEl0 (this->sp_el0); diff --git a/libraries/libmesosphere/source/kern_k_process.cpp b/libraries/libmesosphere/source/kern_k_process.cpp index 789bdbc09..96d82f975 100644 --- a/libraries/libmesosphere/source/kern_k_process.cpp +++ b/libraries/libmesosphere/source/kern_k_process.cpp @@ -299,7 +299,7 @@ namespace ams::kern { /* Setup page table. */ { const bool from_back = (params.flags & ams::svc::CreateProcessFlag_EnableAslr) == 0; - R_TRY(m_page_table.Initialize(static_cast(params.flags), from_back, pool, params.code_address, params.code_num_pages * PageSize, m_system_resource, res_limit)); + R_TRY(m_page_table.Initialize(static_cast(params.flags), from_back, pool, params.code_address, params.code_num_pages * PageSize, m_system_resource, res_limit, this->GetSlabIndex())); } ON_RESULT_FAILURE_2 { m_page_table.Finalize(); }; @@ -378,7 +378,7 @@ namespace ams::kern { /* Setup page table. */ { const bool from_back = (params.flags & ams::svc::CreateProcessFlag_EnableAslr) == 0; - R_TRY(m_page_table.Initialize(static_cast(params.flags), from_back, pool, params.code_address, code_size, m_system_resource, res_limit)); + R_TRY(m_page_table.Initialize(static_cast(params.flags), from_back, pool, params.code_address, code_size, m_system_resource, res_limit, this->GetSlabIndex())); } ON_RESULT_FAILURE_2 { m_page_table.Finalize(); }; diff --git a/mesosphere/kernel/kernel.ld b/mesosphere/kernel/kernel.ld index 779488a35..bcd658572 100644 --- a/mesosphere/kernel/kernel.ld +++ b/mesosphere/kernel/kernel.ld @@ -125,6 +125,8 @@ SECTIONS .gnu.version_r : { *(.gnu.version_r) } :rodata .note.gnu.build-id : { *(.note.gnu.build-id) } :rodata + __rodata_end = .; + /* =========== DATA section =========== */ . = ALIGN(0x1000); __data_start = . ; diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp index faf34576d..10a000c6d 100644 --- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp +++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp @@ -15,6 +15,9 @@ */ #include +extern "C" void __rodata_start(); +extern "C" void __rodata_end(); + extern "C" void __bin_start__(); extern "C" void __bin_end__(); @@ -220,6 +223,31 @@ namespace ams::kern::init { }; static_assert(kern::arch::arm64::init::IsInitialPageAllocator); + void SetupAllTtbr0Entries(KInitialPageTable &init_pt, KInitialPageAllocator &allocator) { + /* Validate that the ttbr0 array is in rodata. */ + const uintptr_t rodata_start = reinterpret_cast(__rodata_start); + const uintptr_t rodata_end = reinterpret_cast(__rodata_end); + MESOSPHERE_INIT_ABORT_UNLESS(rodata_start < rodata_end); + MESOSPHERE_INIT_ABORT_UNLESS(rodata_start <= reinterpret_cast(std::addressof(KPageTable::GetTtbr0Entry(0)))); + MESOSPHERE_INIT_ABORT_UNLESS(reinterpret_cast(std::addressof(KPageTable::GetTtbr0Entry(KPageTable::NumTtbr0Entries))) < rodata_end); + + /* Allocate pages for all ttbr0 entries. */ + for (size_t i = 0; i < KPageTable::NumTtbr0Entries; ++i) { + /* Allocate a page. */ + KPhysicalAddress page = allocator.Allocate(PageSize); + MESOSPHERE_INIT_ABORT_UNLESS(page != Null); + + /* Check that the page is allowed to be a ttbr0 entry. */ + MESOSPHERE_INIT_ABORT_UNLESS((GetInteger(page) & UINT64_C(0xFFFF000000000001)) == 0); + + /* Get the physical address of the ttbr0 entry. */ + const auto ttbr0_phys_ptr = init_pt.GetPhysicalAddress(KVirtualAddress(std::addressof(KPageTable::GetTtbr0Entry(i)))); + + /* Set the entry to the newly allocated page. */ + *reinterpret_cast(GetInteger(ttbr0_phys_ptr)) = (static_cast(i) << 48) | GetInteger(page); + } + } + void FinalizeIdentityMapping(KInitialPageTable &init_pt, KInitialPageAllocator &allocator, u64 phys_to_virt_offset) { /* Create an allocator for identity mapping finalization. */ KInitialPageAllocatorForFinalizeIdentityMapping finalize_allocator(allocator, phys_to_virt_offset); @@ -591,6 +619,9 @@ namespace ams::kern::init { /* Create page table object for use during remaining initialization. */ KInitialPageTable init_pt; + /* Setup all ttbr0 pages. */ + SetupAllTtbr0Entries(init_pt, g_initial_page_allocator); + /* Unmap the identity mapping. */ FinalizeIdentityMapping(init_pt, g_initial_page_allocator, g_phase2_linear_region_phys_to_virt_diff);