From 2fb258ca7e4a16898fe7c20a83465fd271efcf62 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Wed, 7 Apr 2021 00:09:29 -0700 Subject: [PATCH] kern: update KInitialPageTable/KInitialPageAllocator --- .../arm64/init/kern_k_init_page_table.hpp | 241 ++++++++++++++---- .../arch/arm64/kern_cpu_system_registers.hpp | 5 + .../source/arch/arm64/init/kern_init_core.cpp | 6 +- .../kernel_ldr/source/kern_init_loader.cpp | 52 ++-- 4 files changed, 225 insertions(+), 79 deletions(-) diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp index 21fbb28b5..ea53fca94 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp @@ -38,37 +38,67 @@ namespace ams::kern::arch::arm64::init { public: class IPageAllocator { public: - virtual KPhysicalAddress Allocate() { return Null; } - virtual void Free(KPhysicalAddress phys_addr) { /* Nothing to do here. */ (void)(phys_addr); } + virtual KPhysicalAddress Allocate(size_t size) = 0; + virtual void Free(KPhysicalAddress phys_addr, size_t size) = 0; }; - - struct NoClear{}; private: - KPhysicalAddress m_l1_table; + KPhysicalAddress m_l1_tables[2]; + u32 m_num_entries[2]; + public: - constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1, NoClear) : m_l1_table(l1) { /* ... */ } + KInitialPageTable(KVirtualAddress start_address, KVirtualAddress end_address, IPageAllocator &allocator) { + /* Set tables. */ + m_l1_tables[0] = AllocateNewPageTable(allocator); + m_l1_tables[1] = AllocateNewPageTable(allocator); - constexpr ALWAYS_INLINE KInitialPageTable(KPhysicalAddress l1) : KInitialPageTable(l1, NoClear{}) { - ClearNewPageTable(m_l1_table); + /* Set counts. */ + m_num_entries[0] = MaxPageTableEntries; + m_num_entries[1] = ((end_address / L1BlockSize) & (MaxPageTableEntries - 1)) - ((start_address / L1BlockSize) & (MaxPageTableEntries - 1)) + 1; } - constexpr ALWAYS_INLINE uintptr_t GetL1TableAddress() const { - return GetInteger(m_l1_table); + KInitialPageTable() { + /* Set tables. */ + m_l1_tables[0] = util::AlignDown(cpu::GetTtbr0El1(), PageSize); + m_l1_tables[1] = util::AlignDown(cpu::GetTtbr1El1(), PageSize); + + /* Set counts. */ + cpu::TranslationControlRegisterAccessor tcr; + m_num_entries[0] = tcr.GetT0Size() / L1BlockSize; + m_num_entries[1] = tcr.GetT1Size() / L1BlockSize; + + /* Check counts. */ + MESOSPHERE_INIT_ABORT_UNLESS(0 < m_num_entries[0] && m_num_entries[0] <= MaxPageTableEntries); + MESOSPHERE_INIT_ABORT_UNLESS(0 < m_num_entries[1] && m_num_entries[1] <= MaxPageTableEntries); + } + + constexpr ALWAYS_INLINE uintptr_t GetTtbr0L1TableAddress() const { + return GetInteger(m_l1_tables[0]); + } + + constexpr ALWAYS_INLINE uintptr_t GetTtbr1L1TableAddress() const { + return GetInteger(m_l1_tables[1]); } private: - static constexpr ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KPhysicalAddress _l1_table, KVirtualAddress address) { - L1PageTableEntry *l1_table = reinterpret_cast(GetInteger(_l1_table)); - return l1_table + ((GetInteger(address) >> 30) & (MaxPageTableEntries - 1)); + constexpr ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KVirtualAddress address) const { + const size_t index = (GetInteger(address) >> (BITSIZEOF(address) - 1)) & 1; + L1PageTableEntry *l1_table = reinterpret_cast(GetInteger(m_l1_tables[index])); + return l1_table + ((GetInteger(address) / L1BlockSize) & (m_num_entries[index] - 1)); } static constexpr ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KVirtualAddress address) { L2PageTableEntry *l2_table = reinterpret_cast(GetInteger(entry->GetTable())); - return l2_table + ((GetInteger(address) >> 21) & (MaxPageTableEntries - 1)); + return l2_table + ((GetInteger(address) / L2BlockSize) & (MaxPageTableEntries - 1)); } static constexpr ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KVirtualAddress address) { L3PageTableEntry *l3_table = reinterpret_cast(GetInteger(entry->GetTable())); - return l3_table + ((GetInteger(address) >> 12) & (MaxPageTableEntries - 1)); + return l3_table + ((GetInteger(address) / L3BlockSize) & (MaxPageTableEntries - 1)); + } + + static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(IPageAllocator &allocator) { + auto address = allocator.Allocate(PageSize); + ClearNewPageTable(address); + return address; } static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address) { @@ -83,7 +113,7 @@ namespace ams::kern::arch::arm64::init { const KVirtualAddress end_virt_addr = virt_addr + size; size_t count = 0; while (virt_addr < end_virt_addr) { - L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */ if (l1_entry->IsBlock() || l1_entry->IsEmpty()) { @@ -137,7 +167,7 @@ namespace ams::kern::arch::arm64::init { const KVirtualAddress end_virt_addr = virt_addr + size; size_t count = 0; while (virt_addr < end_virt_addr) { - L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); /* If an L1 block is mapped or we're empty, advance by L1BlockSize. */ if (l1_entry->IsBlock() || l1_entry->IsEmpty()) { @@ -194,7 +224,7 @@ namespace ams::kern::arch::arm64::init { } PageTableEntry *GetMappingEntry(KVirtualAddress virt_addr, size_t block_size) { - L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); if (l1_entry->IsBlock()) { MESOSPHERE_INIT_ABORT_UNLESS(block_size == L1BlockSize); @@ -301,7 +331,7 @@ namespace ams::kern::arch::arm64::init { /* Iteratively map pages until the requested region is mapped. */ while (size > 0) { - L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); /* Can we make an L1 block? */ if (util::IsAligned(GetInteger(virt_addr), L1BlockSize) && util::IsAligned(GetInteger(phys_addr), L1BlockSize) && size >= L1BlockSize) { @@ -316,7 +346,7 @@ namespace ams::kern::arch::arm64::init { /* If we don't already have an L2 table, we need to make a new one. */ if (!l1_entry->IsTable()) { - KPhysicalAddress new_table = allocator.Allocate(); + KPhysicalAddress new_table = AllocateNewPageTable(allocator); ClearNewPageTable(new_table); *l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever()); cpu::DataSynchronizationBarrierInnerShareable(); @@ -350,7 +380,7 @@ namespace ams::kern::arch::arm64::init { /* If we don't already have an L3 table, we need to make a new one. */ if (!l2_entry->IsTable()) { - KPhysicalAddress new_table = allocator.Allocate(); + KPhysicalAddress new_table = AllocateNewPageTable(allocator); ClearNewPageTable(new_table); *l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever()); cpu::DataSynchronizationBarrierInnerShareable(); @@ -382,7 +412,7 @@ namespace ams::kern::arch::arm64::init { KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const { /* Get the L1 entry. */ - const L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + const L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); if (l1_entry->IsBlock()) { return l1_entry->GetBlock() + (GetInteger(virt_addr) & (L1BlockSize - 1)); @@ -444,7 +474,7 @@ namespace ams::kern::arch::arm64::init { }; while (virt_addr < end_virt_addr) { - L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); /* If an L1 block is mapped, update. */ if (l1_entry->IsBlock()) { @@ -485,7 +515,7 @@ namespace ams::kern::arch::arm64::init { const KVirtualAddress end_virt_addr = virt_addr + size; while (virt_addr < end_virt_addr) { - L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); /* If an L1 block is mapped, the address isn't free. */ if (l1_entry->IsBlock()) { @@ -534,7 +564,7 @@ namespace ams::kern::arch::arm64::init { /* Iteratively reprotect pages until the requested region is reprotected. */ while (size > 0) { - L1PageTableEntry *l1_entry = GetL1Entry(m_l1_table, virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); /* Check if an L1 block is present. */ if (l1_entry->IsBlock()) { @@ -673,11 +703,18 @@ namespace ams::kern::arch::arm64::init { }; - class KInitialPageAllocator : public KInitialPageTable::IPageAllocator { + class KInitialPageAllocator final : public KInitialPageTable::IPageAllocator { + private: + static constexpr inline size_t FreeUnitSize = BITSIZEOF(u64) * PageSize; + struct FreeListEntry { + FreeListEntry *next; + size_t size; + }; public: struct State { - uintptr_t next_address; - uintptr_t free_bitmap; + uintptr_t start_address; + uintptr_t end_address; + FreeListEntry *free_head; }; private: State m_state; @@ -685,8 +722,8 @@ namespace ams::kern::arch::arm64::init { constexpr ALWAYS_INLINE KInitialPageAllocator() : m_state{} { /* ... */ } ALWAYS_INLINE void Initialize(uintptr_t address) { - m_state.next_address = address + BITSIZEOF(m_state.free_bitmap) * PageSize; - m_state.free_bitmap = ~uintptr_t(); + m_state.start_address = address; + m_state.end_address = address; } ALWAYS_INLINE void InitializeFromState(uintptr_t state_val) { @@ -697,28 +734,134 @@ namespace ams::kern::arch::arm64::init { *out = m_state; m_state = {}; } - public: - virtual KPhysicalAddress Allocate() override { - MESOSPHERE_INIT_ABORT_UNLESS(m_state.next_address != Null); - uintptr_t allocated = m_state.next_address; - if (m_state.free_bitmap != 0) { - u64 index; - uintptr_t mask; - do { - index = KSystemControl::Init::GenerateRandomRange(0, BITSIZEOF(m_state.free_bitmap) - 1); - mask = (static_cast(1) << index); - } while ((m_state.free_bitmap & mask) == 0); - m_state.free_bitmap &= ~mask; - allocated = m_state.next_address - ((BITSIZEOF(m_state.free_bitmap) - index) * PageSize); - } else { - m_state.next_address += PageSize; + private: + bool CanAllocate(size_t align, size_t size) const { + for (auto *cur = m_state.free_head; cur != nullptr; cur = cur->next) { + const uintptr_t cur_last = reinterpret_cast(cur) + cur->size - 1; + const uintptr_t alloc_last = util::AlignUp(reinterpret_cast(cur), align) + size - 1; + if (alloc_last <= cur_last) { + return true; + } } - - ClearPhysicalMemory(allocated, PageSize); - return allocated; + return false; } - /* No need to override free. The default does nothing, and so would we. */ + bool TryAllocate(uintptr_t address, size_t size) { + /* Try to allocate the region. */ + auto **prev_next = std::addressof(m_state.free_head); + for (auto *cur = m_state.free_head; cur != nullptr; prev_next = std::addressof(cur->next), cur = cur->next) { + const uintptr_t cur_start = reinterpret_cast(cur); + const uintptr_t cur_last = cur_start + cur->size - 1; + if (cur_start <= address && address + size - 1 <= cur_last) { + auto *alloc = reinterpret_cast(address); + + /* Perform fragmentation at front. */ + if (cur != alloc) { + prev_next = std::addressof(cur->next); + *alloc = { + .next = cur->next, + .size = cur_start + cur->size - address, + }; + *cur = { + .next = alloc, + .size = address - cur_start, + }; + } + + /* Perform fragmentation at tail. */ + if (alloc->size != size) { + auto *next = reinterpret_cast(address + size); + *next = { + .next = alloc->next, + .size = alloc->size - size, + }; + *alloc = { + .next = next, + .size = size, + }; + } + + *prev_next = alloc->next; + return true; + } + } + + return false; + } + public: + KPhysicalAddress Allocate(size_t align, size_t size) { + /* Ensure that the free list is non-empty. */ + while (!this->CanAllocate(align, size)) { + this->Free(m_state.end_address, FreeUnitSize); + m_state.end_address += FreeUnitSize; + } + + /* Allocate a random address. */ + const uintptr_t aligned_start = util::AlignUp(m_state.start_address, align); + const uintptr_t aligned_end = util::AlignDown(m_state.end_address, align); + const size_t ind_max = ((aligned_end - aligned_start) / align) - 1; + while (true) { + if (const uintptr_t random_address = aligned_start + (KSystemControl::Init::GenerateRandomRange(0, ind_max) * align); this->TryAllocate(random_address, size)) { + return random_address; + } + } + } + + virtual KPhysicalAddress Allocate(size_t size) override { + return this->Allocate(size, size); + } + + virtual void Free(KPhysicalAddress phys_addr, size_t size) override { + auto **prev_next = std::addressof(m_state.free_head); + auto *new_chunk = reinterpret_cast(GetInteger(phys_addr)); + if (auto *cur = m_state.free_head; cur != nullptr) { + const uintptr_t new_start = reinterpret_cast(new_chunk); + const uintptr_t new_end = GetInteger(phys_addr) + size; + while (true) { + /* Attempt coalescing. */ + const uintptr_t cur_start = reinterpret_cast(cur); + const uintptr_t cur_end = cur_start + cur->size; + if (new_start < new_end) { + if (new_end < cur_start) { + *new_chunk = { + .next = cur, + .size = size, + }; + break; + } else if (new_end == cur_start) { + *new_chunk = { + .next = cur->next, + .size = cur->size + size, + }; + break; + } + } else if (cur_end == new_start) { + cur->size += size; + return; + } + + prev_next = std::addressof(cur->next); + if (cur->next != nullptr) { + cur = cur->next; + } else { + *new_chunk = { + .next = nullptr, + .size = size, + }; + cur->next = new_chunk; + return; + } + } + + } else { + *new_chunk = { + .next = nullptr, + .size = size, + }; + } + + *prev_next = new_chunk; + } }; } diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp index 2d4410f84..7a5fe499c 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp @@ -197,6 +197,11 @@ namespace ams::kern::arch::arm64::cpu { public: MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(TranslationControl, tcr_el1) + constexpr ALWAYS_INLINE size_t GetT0Size() const { + const size_t shift_value = this->GetBits(0, 6); + return size_t(1) << (size_t(64) - shift_value); + } + constexpr ALWAYS_INLINE size_t GetT1Size() const { const size_t shift_value = this->GetBits(16, 6); return size_t(1) << (size_t(64) - shift_value); diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp index 6332eb27a..98cd02ed8 100644 --- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp +++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp @@ -50,7 +50,7 @@ namespace ams::kern::init { constexpr size_t StackSize = PageSize; constexpr size_t StackAlign = PageSize; const KVirtualAddress stack_start_virt = KMemoryLayout::GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard(StackSize, StackAlign, KMemoryRegionType_KernelMisc, PageSize); - const KPhysicalAddress stack_start_phys = g_initial_page_allocator.Allocate(); + const KPhysicalAddress stack_start_phys = g_initial_page_allocator.Allocate(PageSize); MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(stack_start_virt), StackSize, type, core_id)); page_table.Map(stack_start_virt, StackSize, stack_start_phys, KernelRwDataAttribute, g_initial_page_allocator); @@ -135,7 +135,7 @@ namespace ams::kern::init { MESOSPHERE_INIT_ABORT_UNLESS((cpu::TranslationControlRegisterAccessor().GetT1Size() / arch::arm64::L1BlockSize) == arch::arm64::MaxPageTableEntries); /* Create page table object for use during initialization. */ - KInitialPageTable ttbr1_table(util::AlignDown(cpu::GetTtbr1El1(), PageSize), KInitialPageTable::NoClear{}); + KInitialPageTable ttbr1_table; /* Initialize the slab allocator counts. */ InitializeSlabResourceCounts(); @@ -382,7 +382,7 @@ namespace ams::kern::init { /* Finalize the page allocator, we're done allocating at this point. */ KInitialPageAllocator::State final_init_page_table_state; g_initial_page_allocator.GetFinalState(std::addressof(final_init_page_table_state)); - const KPhysicalAddress final_init_page_table_end_address = final_init_page_table_state.next_address; + const KPhysicalAddress final_init_page_table_end_address = final_init_page_table_state.end_address; const size_t init_page_table_region_size = GetInteger(final_init_page_table_end_address) - GetInteger(resource_end_phys_addr); /* Insert regions for the initial page table region. */ diff --git a/mesosphere/kernel_ldr/source/kern_init_loader.cpp b/mesosphere/kernel_ldr/source/kern_init_loader.cpp index 0f26fae46..6be53d77b 100644 --- a/mesosphere/kernel_ldr/source/kern_init_loader.cpp +++ b/mesosphere/kernel_ldr/source/kern_init_loader.cpp @@ -28,6 +28,14 @@ namespace ams::kern::init::loader { namespace { + constexpr uintptr_t KernelBaseAlignment = 0x200000; + constexpr uintptr_t KernelBaseRangeStart = 0xFFFFFF8000000000; + constexpr uintptr_t KernelBaseRangeEnd = 0xFFFFFFFFFFE00000; + constexpr uintptr_t KernelBaseRangeLast = KernelBaseRangeEnd - 1; + static_assert(util::IsAligned(KernelBaseRangeStart, KernelBaseAlignment)); + static_assert(util::IsAligned(KernelBaseRangeEnd, KernelBaseAlignment)); + static_assert(KernelBaseRangeStart <= KernelBaseRangeLast); + static_assert(InitialProcessBinarySizeMax <= KernelResourceSize); constexpr size_t InitialPageTableRegionSizeMax = 2_MB; @@ -71,27 +79,24 @@ namespace ams::kern::init::loader { cpu::InvalidateEntireTlb(); } - void SetupInitialIdentityMapping(KInitialPageTable &ttbr1_table, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::IPageAllocator &allocator) { - /* Make a new page table for TTBR0_EL1. */ - KInitialPageTable ttbr0_table(allocator.Allocate()); - + void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::IPageAllocator &allocator) { /* Map in an RWX identity mapping for the kernel. */ constexpr PageTableEntry KernelRWXIdentityAttribute(PageTableEntry::Permission_KernelRWX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); - ttbr0_table.Map(base_address, kernel_size, base_address, KernelRWXIdentityAttribute, allocator); + init_pt.Map(base_address, kernel_size, base_address, KernelRWXIdentityAttribute, allocator); /* Map in an RWX identity mapping for ourselves. */ constexpr PageTableEntry KernelLdrRWXIdentityAttribute(PageTableEntry::Permission_KernelRWX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); const uintptr_t kernel_ldr_base = util::AlignDown(reinterpret_cast(__start__), PageSize); const uintptr_t kernel_ldr_size = util::AlignUp(reinterpret_cast(__end__), PageSize) - kernel_ldr_base; - ttbr0_table.Map(kernel_ldr_base, kernel_ldr_size, kernel_ldr_base, KernelRWXIdentityAttribute, allocator); + init_pt.Map(kernel_ldr_base, kernel_ldr_size, kernel_ldr_base, KernelRWXIdentityAttribute, allocator); /* Map in the page table region as RW- for ourselves. */ constexpr PageTableEntry PageTableRegionRWAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); - ttbr0_table.Map(page_table_region, page_table_region_size, page_table_region, KernelRWXIdentityAttribute, allocator); + init_pt.Map(page_table_region, page_table_region_size, page_table_region, KernelRWXIdentityAttribute, allocator); /* Place the L1 table addresses in the relevant system registers. */ - cpu::SetTtbr0El1(ttbr0_table.GetL1TableAddress()); - cpu::SetTtbr1El1(ttbr1_table.GetL1TableAddress()); + cpu::SetTtbr0El1(init_pt.GetTtbr0L1TableAddress()); + cpu::SetTtbr1El1(init_pt.GetTtbr1L1TableAddress()); /* Setup MAIR_EL1, TCR_EL1. */ /* TODO: Define these bits properly elsewhere, document exactly what each bit set is doing .*/ @@ -115,19 +120,12 @@ namespace ams::kern::init::loader { KVirtualAddress GetRandomKernelBaseAddress(KInitialPageTable &page_table, KPhysicalAddress phys_base_address, size_t kernel_size) { /* Define useful values for random generation. */ - constexpr uintptr_t KernelBaseAlignment = 0x200000; - constexpr uintptr_t KernelBaseRangeMin = 0xFFFFFF8000000000; - constexpr uintptr_t KernelBaseRangeMax = 0xFFFFFFFFFFE00000; - constexpr uintptr_t KernelBaseRangeEnd = KernelBaseRangeMax - 1; - static_assert(util::IsAligned(KernelBaseRangeMin, KernelBaseAlignment)); - static_assert(util::IsAligned(KernelBaseRangeMax, KernelBaseAlignment)); - static_assert(KernelBaseRangeMin <= KernelBaseRangeEnd); const uintptr_t kernel_offset = GetInteger(phys_base_address) % KernelBaseAlignment; /* Repeatedly generate a random virtual address until we get one that's unmapped in the destination page table. */ while (true) { - const uintptr_t random_kaslr_slide = KSystemControl::Init::GenerateRandomRange(KernelBaseRangeMin / KernelBaseAlignment, KernelBaseRangeEnd / KernelBaseAlignment); + const uintptr_t random_kaslr_slide = KSystemControl::Init::GenerateRandomRange(KernelBaseRangeStart / KernelBaseAlignment, KernelBaseRangeLast / KernelBaseAlignment); const KVirtualAddress kernel_region_start = random_kaslr_slide * KernelBaseAlignment; const KVirtualAddress kernel_region_end = kernel_region_start + util::AlignUp(kernel_offset + kernel_size, KernelBaseAlignment); const size_t kernel_region_size = GetInteger(kernel_region_end) - GetInteger(kernel_region_start); @@ -138,7 +136,7 @@ namespace ams::kern::init::loader { } /* Make sure that the region stays within our intended bounds. */ - if (kernel_region_end > KernelBaseRangeMax) { + if (kernel_region_end > KernelBaseRangeEnd) { continue; } @@ -201,28 +199,28 @@ namespace ams::kern::init::loader { g_initial_page_allocator.Initialize(ini_end_address); /* Make a new page table for TTBR1_EL1. */ - KInitialPageTable ttbr1_table(g_initial_page_allocator.Allocate()); + KInitialPageTable init_pt(KernelBaseRangeStart, KernelBaseRangeLast, g_initial_page_allocator); /* Setup initial identity mapping. TTBR1 table passed by reference. */ - SetupInitialIdentityMapping(ttbr1_table, base_address, bss_end_offset, ini_end_address, InitialPageTableRegionSizeMax, g_initial_page_allocator); + SetupInitialIdentityMapping(init_pt, base_address, bss_end_offset, ini_end_address, InitialPageTableRegionSizeMax, g_initial_page_allocator); /* Generate a random slide for the kernel's base address. */ - const KVirtualAddress virtual_base_address = GetRandomKernelBaseAddress(ttbr1_table, base_address, bss_end_offset); + const KVirtualAddress virtual_base_address = GetRandomKernelBaseAddress(init_pt, base_address, bss_end_offset); /* Map kernel .text as R-X. */ constexpr PageTableEntry KernelTextAttribute(PageTableEntry::Permission_KernelRX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); - ttbr1_table.Map(virtual_base_address + rx_offset, rx_end_offset - rx_offset, base_address + rx_offset, KernelTextAttribute, g_initial_page_allocator); + init_pt.Map(virtual_base_address + rx_offset, rx_end_offset - rx_offset, base_address + rx_offset, KernelTextAttribute, g_initial_page_allocator); /* Map kernel .rodata and .rwdata as RW-. */ /* Note that we will later reprotect .rodata as R-- */ constexpr PageTableEntry KernelRoDataAttribute(PageTableEntry::Permission_KernelR, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); - ttbr1_table.Map(virtual_base_address + ro_offset, ro_end_offset - ro_offset, base_address + ro_offset, KernelRwDataAttribute, g_initial_page_allocator); - ttbr1_table.Map(virtual_base_address + rw_offset, bss_end_offset - rw_offset, base_address + rw_offset, KernelRwDataAttribute, g_initial_page_allocator); + init_pt.Map(virtual_base_address + ro_offset, ro_end_offset - ro_offset, base_address + ro_offset, KernelRwDataAttribute, g_initial_page_allocator); + init_pt.Map(virtual_base_address + rw_offset, bss_end_offset - rw_offset, base_address + rw_offset, KernelRwDataAttribute, g_initial_page_allocator); /* Physically randomize the kernel region. */ /* NOTE: Nintendo does this only on 10.0.0+ */ - ttbr1_table.PhysicallyRandomize(virtual_base_address + rx_offset, bss_end_offset - rx_offset, true); + init_pt.PhysicallyRandomize(virtual_base_address + rx_offset, bss_end_offset - rx_offset, true); /* Clear kernel .bss. */ std::memset(GetVoidPointer(virtual_base_address + bss_offset), 0, bss_end_offset - bss_offset); @@ -237,14 +235,14 @@ namespace ams::kern::init::loader { Elf::CallInitArrayFuncs(GetInteger(virtual_base_address) + init_array_offset, GetInteger(virtual_base_address) + init_array_end_offset); /* Reprotect .rodata as R-- */ - ttbr1_table.Reprotect(virtual_base_address + ro_offset, ro_end_offset - ro_offset, KernelRwDataAttribute, KernelRoDataAttribute); + init_pt.Reprotect(virtual_base_address + ro_offset, ro_end_offset - ro_offset, KernelRwDataAttribute, KernelRoDataAttribute); /* Return the difference between the random virtual base and the physical base. */ return GetInteger(virtual_base_address) - base_address; } KPhysicalAddress AllocateKernelInitStack() { - return g_initial_page_allocator.Allocate() + PageSize; + return g_initial_page_allocator.Allocate(PageSize) + PageSize; } uintptr_t GetFinalPageAllocatorState() {