From f3b532070ba5f7f4833ee72960f85a86119c56f3 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Tue, 19 Oct 2021 01:20:28 -0700 Subject: [PATCH] kern: update scheduler for 13.0.0 change, fix some ctz/clz bugs --- .../arm/kern_generic_interrupt_controller.hpp | 3 +- .../arch/arm64/kern_k_page_table_impl.hpp | 3 +- .../mesosphere/kern_k_capabilities.hpp | 36 ++----------------- .../mesosphere/kern_k_priority_queue.hpp | 2 +- .../include/mesosphere/kern_k_process.hpp | 2 ++ .../libmesosphere/source/kern_k_scheduler.cpp | 12 ++++--- .../libmesosphere/source/kern_k_thread.cpp | 2 +- .../include/vapours/util/util_alignment.hpp | 5 --- .../include/vapours/util/util_bitset.hpp | 13 +++---- 9 files changed, 22 insertions(+), 56 deletions(-) diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm/kern_generic_interrupt_controller.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm/kern_generic_interrupt_controller.hpp index c2b02622c..146f38f87 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm/kern_generic_interrupt_controller.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm/kern_generic_interrupt_controller.hpp @@ -248,8 +248,9 @@ namespace ams::kern::arch::arm { return id; } private: - static constexpr size_t PriorityShift = BITSIZEOF(u8) - __builtin_ctz(NumPriorityLevels); + static constexpr size_t PriorityShift = BITSIZEOF(u8) - util::CountTrailingZeros(NumPriorityLevels); static_assert(PriorityShift < BITSIZEOF(u8)); + static_assert(util::IsPowerOfTwo(NumPriorityLevels)); static constexpr ALWAYS_INLINE u8 ToGicPriorityValue(s32 level) { return (level << PriorityShift) | ((1 << PriorityShift) - 1); diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp index d29ef35c9..9f61f29e7 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_impl.hpp @@ -42,10 +42,11 @@ namespace ams::kern::arch::arm64 { const L3PageTableEntry *l3_entry; }; private: - static constexpr size_t PageBits = __builtin_ctzll(PageSize); + static constexpr size_t PageBits = util::CountTrailingZeros(PageSize); static constexpr size_t NumLevels = 3; static constexpr size_t LevelBits = 9; static_assert(NumLevels > 0); + static_assert(PageBits == 12); template static constexpr ALWAYS_INLINE u64 GetBits(u64 value) { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_capabilities.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_capabilities.hpp index f9d2d9cc1..f1c53d98c 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_capabilities.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_capabilities.hpp @@ -55,48 +55,16 @@ namespace ams::kern { return static_cast(type) + 1; } - static constexpr u32 CountTrailingZero(u32 flag) { - for (u32 i = 0; i < BITSIZEOF(u32); i++) { - if (flag & (1u << i)) { - return i; - } - } - return BITSIZEOF(u32); - } - - static constexpr u32 GetCapabilityId(CapabilityType type) { - const u32 flag = GetCapabilityFlag(type); - if (std::is_constant_evaluated()) { - return CountTrailingZero(flag); - } else { - return static_cast(__builtin_ctz(flag)); - } - } - template using Field = util::BitPack32::Field; #define DEFINE_FIELD(name, prev, ...) using name = Field template - static constexpr inline u32 CapabilityFlag = []() -> u32 { - return static_cast(Type) + 1; - }(); + static constexpr inline u32 CapabilityFlag = static_cast(Type) + 1; template - static constexpr inline u32 CapabilityId = []() -> u32 { - const u32 flag = static_cast(Type) + 1; - if (std::is_constant_evaluated()) { - for (u32 i = 0; i < BITSIZEOF(u32); i++) { - if (flag & (1u << i)) { - return i; - } - } - return BITSIZEOF(u32); - } else { - return __builtin_ctz(flag); - } - }(); + static constexpr inline u32 CapabilityId = util::CountTrailingZeros(CapabilityFlag); struct CorePriority { using IdBits = Field<0, CapabilityId + 1>; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_priority_queue.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_priority_queue.hpp index aeea928b3..6a14b85db 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_priority_queue.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_priority_queue.hpp @@ -235,7 +235,7 @@ namespace ams::kern { KPriorityQueueImpl m_suggested_queue; private: constexpr ALWAYS_INLINE void ClearAffinityBit(u64 &affinity, s32 core) { - affinity &= ~(u64(1ul) << core); + affinity &= ~(UINT64_C(1) << core); } constexpr ALWAYS_INLINE s32 GetNextCore(u64 &affinity) { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp index 7453fd2e6..4246f1680 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -286,6 +286,8 @@ namespace ams::kern { constexpr void *GetProcessLocalRegionHeapAddress() const { return m_plr_heap_address; } + KThread *GetExceptionThread() const { return m_exception_thread; } + void AddCpuTime(s64 diff) { m_cpu_time += diff; } s64 GetCpuTime() { return m_cpu_time; } diff --git a/libraries/libmesosphere/source/kern_k_scheduler.cpp b/libraries/libmesosphere/source/kern_k_scheduler.cpp index 21833ef71..417eb324c 100644 --- a/libraries/libmesosphere/source/kern_k_scheduler.cpp +++ b/libraries/libmesosphere/source/kern_k_scheduler.cpp @@ -126,11 +126,13 @@ namespace ams::kern { for (size_t core_id = 0; core_id < cpu::NumCores; core_id++) { KThread *top_thread = priority_queue.GetScheduledFront(core_id); if (top_thread != nullptr) { - /* If the thread has no waiters, we need to check if the process has a thread pinned. */ - if (top_thread->GetNumKernelWaiters() == 0) { - if (KProcess *parent = top_thread->GetOwnerProcess(); parent != nullptr) { - if (KThread *pinned = parent->GetPinnedThread(core_id); pinned != nullptr && pinned != top_thread) { - /* We prefer our parent's pinned thread if possible. However, we also don't want to schedule un-runnable threads. */ + /* We need to check if the thread's process has a pinned thread. */ + if (KProcess *parent = top_thread->GetOwnerProcess(); parent != nullptr) { + /* Check that there's a pinned thread other than the current top thread. */ + if (KThread *pinned = parent->GetPinnedThread(core_id); pinned != nullptr && pinned != top_thread) { + /* We need to prefer threads with kernel waiters to the pinned thread. */ + if (top_thread->GetNumKernelWaiters() == 0 && top_thread != parent->GetExceptionThread()) { + /* If the pinned thread is runnable, use it. */ if (pinned->GetRawState() == KThread::ThreadState_Runnable) { top_thread = pinned; } else { diff --git a/libraries/libmesosphere/source/kern_k_thread.cpp b/libraries/libmesosphere/source/kern_k_thread.cpp index 3f19f8880..2d88e9a74 100644 --- a/libraries/libmesosphere/source/kern_k_thread.cpp +++ b/libraries/libmesosphere/source/kern_k_thread.cpp @@ -441,7 +441,7 @@ namespace ams::kern { m_base_priority_on_unpin = old_base_priority; /* Set base priority to higher than any possible process priority. */ - m_base_priority = std::min(old_base_priority, __builtin_ctzll(this->GetOwnerProcess()->GetPriorityMask())); + m_base_priority = std::min(old_base_priority, __builtin_ctzll(this->GetOwnerProcess()->GetPriorityMask()) - 1); RestorePriority(this); } diff --git a/libraries/libvapours/include/vapours/util/util_alignment.hpp b/libraries/libvapours/include/vapours/util/util_alignment.hpp index 80f0bb3b9..2e1ed5e09 100644 --- a/libraries/libvapours/include/vapours/util/util_alignment.hpp +++ b/libraries/libvapours/include/vapours/util/util_alignment.hpp @@ -78,9 +78,4 @@ namespace ams::util { return IsAligned(reinterpret_cast(value), alignment); } - template requires std::integral && std::integral - constexpr ALWAYS_INLINE T DivideUp(T x, U y) { - return (x + (y - 1)) / y; - } - } diff --git a/libraries/libvapours/include/vapours/util/util_bitset.hpp b/libraries/libvapours/include/vapours/util/util_bitset.hpp index 70919a29a..214977742 100644 --- a/libraries/libvapours/include/vapours/util/util_bitset.hpp +++ b/libraries/libvapours/include/vapours/util/util_bitset.hpp @@ -17,6 +17,7 @@ #pragma once #include #include +#include namespace ams::util { @@ -30,14 +31,10 @@ namespace ams::util { static_assert(sizeof(Storage) <= sizeof(u64)); static constexpr size_t FlagsPerWord = BITSIZEOF(Storage); - static constexpr size_t NumWords = util::AlignUp(N, FlagsPerWord) / FlagsPerWord; - - static constexpr ALWAYS_INLINE auto CountLeadingZeroImpl(Storage word) { - return __builtin_clzll(static_cast(word)) - (BITSIZEOF(unsigned long long) - FlagsPerWord); - } + static constexpr size_t NumWords = util::DivideUp(N, FlagsPerWord); static constexpr ALWAYS_INLINE Storage GetBitMask(size_t bit) { - return Storage(1) << (FlagsPerWord - 1 - bit); + return static_cast(1) << (FlagsPerWord - 1 - bit); } private: Storage m_words[NumWords]; @@ -55,7 +52,7 @@ namespace ams::util { constexpr ALWAYS_INLINE size_t CountLeadingZero() const { for (size_t i = 0; i < NumWords; i++) { if (m_words[i]) { - return FlagsPerWord * i + CountLeadingZeroImpl(m_words[i]); + return FlagsPerWord * i + util::CountLeadingZeros(m_words[i]); } } return FlagsPerWord * NumWords; @@ -68,7 +65,7 @@ namespace ams::util { word &= GetBitMask(n % FlagsPerWord) - 1; } if (word) { - return FlagsPerWord * i + CountLeadingZeroImpl(word); + return FlagsPerWord * i + util::CountLeadingZeros(word); } } return FlagsPerWord * NumWords;