diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp index e9ca675ec..c3693bfac 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp @@ -172,10 +172,8 @@ namespace ams::kern::arch::arm64::cpu { /* Cache management helpers. */ void ClearPageToZeroImpl(void *); - void FlushEntireDataCacheSharedForInit(); - void FlushEntireDataCacheLocalForInit(); - void InvalidateEntireInstructionCacheForInit(); void StoreEntireCacheForInit(); + void FlushEntireCacheForInit(); void FlushEntireDataCache(); diff --git a/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp b/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp index 3da383f02..b5ebee323 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp @@ -262,27 +262,6 @@ namespace ams::kern::arch::arm64::cpu { __asm__ __volatile__("dc csw, %[v]" :: [v]"r"(sw_value) : "memory"); } - template - ALWAYS_INLINE void PerformCacheOperationBySetWayShared(F f) { - CacheLineIdRegisterAccessor clidr_el1; - const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency(); - const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); - - for (int level = levels_of_coherency; level >= levels_of_unification; level--) { - PerformCacheOperationBySetWayImpl(level, f); - } - } - - template - ALWAYS_INLINE void PerformCacheOperationBySetWayLocal(F f) { - CacheLineIdRegisterAccessor clidr_el1; - const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); - - for (int level = levels_of_unification - 1; level >= 0; level--) { - PerformCacheOperationBySetWayImpl(level, f); - } - } - void StoreDataCacheBySetWay(int level) { PerformCacheOperationBySetWayImpl(level, StoreDataCacheLineBySetWayImpl); cpu::DataSynchronizationBarrier(); @@ -361,24 +340,63 @@ namespace ams::kern::arch::arm64::cpu { } - void FlushEntireDataCacheSharedForInit() { - return PerformCacheOperationBySetWayShared(FlushDataCacheLineBySetWayImpl); + void StoreEntireCacheForInit() { + /* Store local. */ + { + CacheLineIdRegisterAccessor clidr_el1; + const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); + + for (int level = 0; level != levels_of_unification; ++level) { + PerformCacheOperationBySetWayImpl(level, StoreDataCacheLineBySetWayImpl); + } + } + + /* Store shared. */ + { + CacheLineIdRegisterAccessor clidr_el1; + const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency(); + const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); + + for (int level = levels_of_unification; level <= levels_of_coherency; ++level) { + PerformCacheOperationBySetWayImpl(level, StoreDataCacheLineBySetWayImpl); + } + } + + /* Data synchronization barrier. */ + DataSynchronizationBarrierInnerShareable(); + + /* Invalidate instruction cache. */ + InvalidateEntireInstructionCacheLocalImpl(); + + /* Ensure local instruction consistency. */ + DataSynchronizationBarrierInnerShareable(); + InstructionMemoryBarrier(); } - void FlushEntireDataCacheLocalForInit() { - return PerformCacheOperationBySetWayLocal(FlushDataCacheLineBySetWayImpl); - } + void FlushEntireCacheForInit() { + /* Flush data cache. */ + { + /* Get levels of coherence/unificaiton. */ + CacheLineIdRegisterAccessor clidr_el1; + const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency(); - void InvalidateEntireInstructionCacheForInit() { + /* Store cache from L1 up to (level of coherence - 1). */ + for (int level = 0; level < levels_of_coherency - 1; ++level) { + PerformCacheOperationBySetWayImpl(level, StoreDataCacheLineBySetWayImpl); + } + + /* Flush cache from (level of coherence - 1) down to L0. */ + for (int level = levels_of_coherency; level > 0; --level) { + PerformCacheOperationBySetWayImpl(level - 1, FlushDataCacheLineBySetWayImpl); + } + } + + /* Invalidate instruction cache. */ InvalidateEntireInstructionCacheLocalImpl(); EnsureInstructionConsistency(); - } - void StoreEntireCacheForInit() { - PerformCacheOperationBySetWayLocal(StoreDataCacheLineBySetWayImpl); - PerformCacheOperationBySetWayShared(StoreDataCacheLineBySetWayImpl); - DataSynchronizationBarrierInnerShareable(); - InvalidateEntireInstructionCacheForInit(); + /* Invalidate entire TLB. */ + InvalidateEntireTlb(); } void FlushEntireDataCache() { diff --git a/mesosphere/kernel/source/arch/arm64/init/start.s b/mesosphere/kernel/source/arch/arm64/init/start.s index 6b513d0eb..281db1509 100644 --- a/mesosphere/kernel/source/arch/arm64/init/start.s +++ b/mesosphere/kernel/source/arch/arm64/init/start.s @@ -383,20 +383,20 @@ _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv: /* const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); */ ubfx x10, x10, #0x15, 3 - /* int level = levels_of_unification - 1 */ - sub w9, w10, #1 + /* int level = 0 */ + mov x9, xzr - /* while (level >= 0) { */ + /* while (level <= levels_of_unification) { */ begin_flush_cache_local_loop: - cmn w9, #1 + cmp x9, x10 b.eq done_flush_cache_local_loop /* FlushEntireDataCacheImplWithoutStack(level); */ mov w0, w9 bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv - /* level--; */ - sub w9, w9, #1 + /* level++; */ + add w9, w9, #1 /* } */ b begin_flush_cache_local_loop @@ -416,23 +416,23 @@ _ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv: /* CacheLineIdAccessor clidr_el1; */ mrs x10, clidr_el1 /* const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency(); */ - ubfx x9, x10, #0x18, 3 + ubfx x9, x10, #0x15, 3 /* const int levels_of_unification = clidr_el1.GetLevelsOfUnification(); */ - ubfx x10, x10, #0x15, 3 + ubfx x10, x10, #0x18, 3 - /* int level = levels_of_coherency */ + /* int level = levels_of_unification */ - /* while (level >= levels_of_unification) { */ + /* while (level <= levels_of_coherency) { */ begin_flush_cache_shared_loop: - cmp w10, w9 - b.gt done_flush_cache_shared_loop + cmp w9, w10 + b.hi done_flush_cache_shared_loop /* FlushEntireDataCacheImplWithoutStack(level); */ mov w0, w9 bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv - /* level--; */ - sub w9, w9, #1 + /* level++; */ + add w9, w9, #1 /* } */ b begin_flush_cache_shared_loop diff --git a/mesosphere/kernel_ldr/source/kern_init_loader.cpp b/mesosphere/kernel_ldr/source/kern_init_loader.cpp index 1eae25f16..c283013a5 100644 --- a/mesosphere/kernel_ldr/source/kern_init_loader.cpp +++ b/mesosphere/kernel_ldr/source/kern_init_loader.cpp @@ -59,26 +59,6 @@ namespace ams::kern::init::loader { } } - void EnsureEntireDataCacheFlushed() { - /* Flush shared cache. */ - cpu::FlushEntireDataCacheSharedForInit(); - cpu::DataSynchronizationBarrier(); - - /* Flush local cache. */ - cpu::FlushEntireDataCacheLocalForInit(); - cpu::DataSynchronizationBarrier(); - - /* Flush shared cache. */ - cpu::FlushEntireDataCacheSharedForInit(); - cpu::DataSynchronizationBarrier(); - - /* Invalidate entire instruction cache. */ - cpu::InvalidateEntireInstructionCacheForInit(); - - /* Invalidate entire TLB. */ - cpu::InvalidateEntireTlb(); - } - void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::IPageAllocator &allocator) { /* Map in an RWX identity mapping for the kernel. */ constexpr PageTableEntry KernelRWXIdentityAttribute(PageTableEntry::Permission_KernelRWX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); @@ -109,7 +89,7 @@ namespace ams::kern::init::loader { PerformBoardSpecificSetup(); /* Ensure that the entire cache is flushed. */ - EnsureEntireDataCacheFlushed(); + cpu::FlushEntireCacheForInit(); /* Setup SCTLR_EL1. */ /* TODO: Define these bits properly elsewhere, document exactly what each bit set is doing .*/