From 8a661cee6e78a1e3715d2701671d294a8e66598b Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Sun, 24 Oct 2021 17:00:05 -0700 Subject: [PATCH] kern: devirtualize KAutoObjectWithList::GetId() --- .../libmesosphere/include/mesosphere.hpp | 3 + .../include/mesosphere/kern_k_auto_object.hpp | 27 +-- .../kern_k_auto_object_container.hpp | 166 ++++++++++++------ .../mesosphere/kern_k_auto_object_impls.hpp | 163 +++++++++++++++++ .../include/mesosphere/kern_k_class_token.hpp | 65 +++++++ .../include/mesosphere/kern_k_process.hpp | 3 +- .../include/mesosphere/kern_k_thread.hpp | 13 +- .../include/mesosphere/kern_slab_helpers.hpp | 8 +- 8 files changed, 351 insertions(+), 97 deletions(-) create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp index d594f750c..560831c3e 100644 --- a/libraries/libmesosphere/include/mesosphere.hpp +++ b/libraries/libmesosphere/include/mesosphere.hpp @@ -90,3 +90,6 @@ /* Main functionality. */ #include + +/* Deferred includes. */ +#include diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp index 8c2b8bca3..72f8ec06b 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp @@ -230,8 +230,6 @@ namespace ams::kern { } }; - class KAutoObjectWithListContainer; - class KAutoObjectWithListBase : public KAutoObject { private: void *m_alignment_forcer_unused[0]; @@ -243,6 +241,7 @@ namespace ams::kern { class KAutoObjectWithList : public KAutoObjectWithListBase { private: + template friend class KAutoObjectWithListContainer; private: util::IntrusiveRedBlackTreeNode m_list_node; @@ -250,28 +249,8 @@ namespace ams::kern { constexpr ALWAYS_INLINE KAutoObjectWithList(util::ConstantInitializeTag) : KAutoObjectWithListBase(util::ConstantInitialize), m_list_node(util::ConstantInitialize) { /* ... */ } ALWAYS_INLINE explicit KAutoObjectWithList() { /* ... */ } public: - using RedBlackKeyType = u64; - - static constexpr ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const RedBlackKeyType &v) { return v; } - static constexpr ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const KAutoObjectWithList &v) { return v.GetId(); } - - template requires (std::same_as || std::same_as) - static ALWAYS_INLINE int Compare(const T &lhs, const KAutoObjectWithList &rhs) { - const u64 lid = GetRedBlackKey(lhs); - const u64 rid = GetRedBlackKey(rhs); - - if (lid < rid) { - return -1; - } else if (lid > rid) { - return 1; - } else { - return 0; - } - } - public: - virtual u64 GetId() const { - return reinterpret_cast(this); - } + /* NOTE: This is virtual in Nintendo's kernel. */ + u64 GetId() const; }; template requires std::derived_from diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp index d8e745a0a..a6ddac400 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp @@ -20,77 +20,129 @@ namespace ams::kern { - class KAutoObjectWithListContainer { - NON_COPYABLE(KAutoObjectWithListContainer); - NON_MOVEABLE(KAutoObjectWithListContainer); - public: - using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::m_list_node>::TreeType; - public: - class ListAccessor : public KScopedLightLock { - private: - ListType &m_list; - public: - explicit ListAccessor(KAutoObjectWithListContainer *container) : KScopedLightLock(container->m_lock), m_list(container->m_object_list) { /* ... */ } - explicit ListAccessor(KAutoObjectWithListContainer &container) : KScopedLightLock(container.m_lock), m_list(container.m_object_list) { /* ... */ } + namespace impl { - ALWAYS_INLINE typename ListType::iterator begin() const { - return m_list.begin(); - } + template + struct GetAutoObjectWithListComparator; - ALWAYS_INLINE typename ListType::iterator end() const { - return m_list.end(); - } + class KAutoObjectWithListContainerBase { + NON_COPYABLE(KAutoObjectWithListContainerBase); + NON_MOVEABLE(KAutoObjectWithListContainerBase); + protected: + template + class ListAccessorImpl { + NON_COPYABLE(ListAccessorImpl); + NON_MOVEABLE(ListAccessorImpl); + private: + KScopedLightLock m_lk; + ListType &m_list; + public: + explicit ALWAYS_INLINE ListAccessorImpl(KAutoObjectWithListContainerBase *container, ListType &list) : m_lk(container->m_lock), m_list(list) { /* ... */ } + explicit ALWAYS_INLINE ListAccessorImpl(KAutoObjectWithListContainerBase &container, ListType &list) : m_lk(container.m_lock), m_list(list) { /* ... */ } - ALWAYS_INLINE typename ListType::iterator find(typename ListType::const_reference ref) const { - return m_list.find(ref); - } + ALWAYS_INLINE ~ListAccessorImpl() { /* ... */ } - ALWAYS_INLINE typename ListType::iterator find_key(typename ListType::const_key_reference ref) const { - return m_list.find_key(ref); - } - }; + ALWAYS_INLINE typename ListType::iterator begin() const { + return m_list.begin(); + } - friend class ListAccessor; - private: - KLightLock m_lock; - ListType m_object_list; - public: - constexpr KAutoObjectWithListContainer() : m_lock(), m_object_list() { MESOSPHERE_ASSERT_THIS(); } + ALWAYS_INLINE typename ListType::iterator end() const { + return m_list.end(); + } - void Initialize() { MESOSPHERE_ASSERT_THIS(); } - void Finalize() { MESOSPHERE_ASSERT_THIS(); } + ALWAYS_INLINE typename ListType::iterator find(typename ListType::const_reference ref) const { + return m_list.find(ref); + } - void Register(KAutoObjectWithList *obj) { - MESOSPHERE_ASSERT_THIS(); + ALWAYS_INLINE typename ListType::iterator find_key(typename ListType::const_key_reference ref) const { + return m_list.find_key(ref); + } + }; - KScopedLightLock lk(m_lock); + template + friend class ListAccessorImpl; + private: + KLightLock m_lock; + protected: + constexpr KAutoObjectWithListContainerBase() : m_lock() { /* ... */ } - m_object_list.insert(*obj); - } + ALWAYS_INLINE void InitializeImpl() { MESOSPHERE_ASSERT_THIS(); } + ALWAYS_INLINE void FinalizeImpl() { MESOSPHERE_ASSERT_THIS(); } - void Unregister(KAutoObjectWithList *obj) { - MESOSPHERE_ASSERT_THIS(); + template + void RegisterImpl(KAutoObjectWithList *obj, ListType &list) { + MESOSPHERE_ASSERT_THIS(); - KScopedLightLock lk(m_lock); + KScopedLightLock lk(m_lock); - m_object_list.erase(m_object_list.iterator_to(*obj)); - } - - template requires (std::derived_from && requires (const T &t) { { t.GetOwner() } -> std::convertible_to; }) - size_t GetOwnedCount(const KProcess *owner) { - MESOSPHERE_ASSERT_THIS(); - - KScopedLightLock lk(m_lock); - - size_t count = 0; - - for (const auto &obj : m_object_list) { - if (const T * const derived = obj.DynamicCast(); derived != nullptr && derived->GetOwner() == owner) { - ++count; - } + list.insert(*obj); } - return count; + template + void UnregisterImpl(KAutoObjectWithList *obj, ListType &list) { + MESOSPHERE_ASSERT_THIS(); + + KScopedLightLock lk(m_lock); + + list.erase(list.iterator_to(*obj)); + } + + template + size_t GetOwnedCountImpl(const KProcess *owner, ListType &list) { + MESOSPHERE_ASSERT_THIS(); + + KScopedLightLock lk(m_lock); + + size_t count = 0; + + for (const auto &obj : list) { + AMS_AUDIT(obj.DynamicCast() != nullptr); + if (static_cast(obj).GetOwner() == owner) { + ++count; + } + } + + return count; + } + }; + + struct DummyKAutoObjectWithListComparator { + static NOINLINE int Compare(KAutoObjectWithList &lhs, KAutoObjectWithList &rhs) { + AMS_ASSUME(false); + } + }; + + } + + template + class KAutoObjectWithListContainer : public impl::KAutoObjectWithListContainerBase { + private: + using Base = impl::KAutoObjectWithListContainerBase; + public: + class ListAccessor; + friend class ListAccessor; + + template + using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::m_list_node>::TreeType; + + using DummyListType = ListType; + private: + DummyListType m_dummy_object_list; + public: + constexpr ALWAYS_INLINE KAutoObjectWithListContainer() : Base(), m_dummy_object_list() { static_assert(std::derived_from); } + + ALWAYS_INLINE void Initialize() { return this->InitializeImpl(); } + ALWAYS_INLINE void Finalize() { return this->FinalizeImpl(); } + + void Register(T *obj); + void Unregister(T *obj); + + private: + size_t GetOwnedCountChecked(const KProcess *owner); + public: + template requires (std::same_as && requires (const U &u) { { u.GetOwner() } -> std::convertible_to; }) + ALWAYS_INLINE size_t GetOwnedCount(const KProcess *owner) { + return this->GetOwnedCountChecked(owner); } }; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp new file mode 100644 index 000000000..58d60da89 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp @@ -0,0 +1,163 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::kern { + + /* NOTE: This header is included after all other KAutoObjects. */ + namespace impl { + + template requires std::derived_from + consteval bool IsAutoObjectInheritanceValidImpl() { + #define CLASS_TOKEN_HANDLER(CLASSNAME) \ + if constexpr (std::same_as) { \ + if (T::GetStaticTypeObj().GetClassToken() != ::ams::kern::ClassToken) { \ + return false; \ + } \ + } else { \ + if (T::GetStaticTypeObj().IsDerivedFrom(CLASSNAME::GetStaticTypeObj()) != std::derived_from) { \ + return false; \ + } \ + } + + FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER) + #undef CLASS_TOKEN_HANDLER + + return true; + } + + consteval bool IsEveryAutoObjectInheritanceValid() { + #define CLASS_TOKEN_HANDLER(CLASSNAME) if (!IsAutoObjectInheritanceValidImpl()) { return false; } + FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER) + #undef CLASS_TOKEN_HANDLER + + return true; + } + + static_assert(IsEveryAutoObjectInheritanceValid()); + + template + concept IsAutoObjectWithSpecializedGetId = std::derived_from && requires (const T &t, const KAutoObjectWithList &l) { + { t.GetIdImpl() } -> std::same_as; + }; + + template + struct AutoObjectWithListComparatorImpl { + using RedBlackKeyType = u64; + + static ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const RedBlackKeyType &v) { return v; } + + static ALWAYS_INLINE RedBlackKeyType GetRedBlackKey(const KAutoObjectWithList &v) { + if constexpr (IsAutoObjectWithSpecializedGetId) { + return static_cast(v).GetIdImpl(); + } else { + return reinterpret_cast(std::addressof(v)); + } + } + + template requires (std::same_as || std::same_as) + static ALWAYS_INLINE int Compare(const U &lhs, const KAutoObjectWithList &rhs) { + const u64 lid = GetRedBlackKey(lhs); + const u64 rid = GetRedBlackKey(rhs); + + if (lid < rid) { + return -1; + } else if (lid > rid) { + return 1; + } else { + return 0; + } + } + }; + + template + using AutoObjectWithListComparator = AutoObjectWithListComparatorImpl, T, KAutoObjectWithList>::type>; + + template + using TrueObjectContainerListType = typename KAutoObjectWithListContainer::ListType>; + + template + ALWAYS_INLINE TrueObjectContainerListType &GetTrueObjectContainerList(typename KAutoObjectWithListContainer::DummyListType &l) { + static_assert(alignof(l) == alignof(impl::TrueObjectContainerListType)); + static_assert(sizeof(l) == sizeof(impl::TrueObjectContainerListType)); + return *reinterpret_cast *>(std::addressof(l)); + } + + } + + ALWAYS_INLINE void KAutoObject::ScheduleDestruction() { + MESOSPHERE_ASSERT_THIS(); + + /* Set our object to destroy. */ + m_next_closed_object = GetCurrentThread().GetClosedObject(); + + /* Set ourselves as the thread's next object to destroy. */ + GetCurrentThread().SetClosedObject(this); + } + + template + class KAutoObjectWithListContainer::ListAccessor : public impl::KAutoObjectWithListContainerBase::ListAccessorImpl> { + NON_COPYABLE(ListAccessor); + NON_MOVEABLE(ListAccessor); + private: + using BaseListAccessor = impl::KAutoObjectWithListContainerBase::ListAccessorImpl>; + public: + explicit ALWAYS_INLINE ListAccessor(KAutoObjectWithListContainer *container) : BaseListAccessor(container, impl::GetTrueObjectContainerList(container->m_dummy_object_list)) { /* ... */ } + explicit ALWAYS_INLINE ListAccessor(KAutoObjectWithListContainer &container) : BaseListAccessor(container, impl::GetTrueObjectContainerList(container.m_dummy_object_list)) { /* ... */ } + + ALWAYS_INLINE ~ListAccessor() { /* ... */ } + }; + + + template + ALWAYS_INLINE void KAutoObjectWithListContainer::Register(T *obj) { + return this->RegisterImpl(obj, impl::GetTrueObjectContainerList(m_dummy_object_list)); + } + + template + ALWAYS_INLINE void KAutoObjectWithListContainer::Unregister(T *obj) { + return this->UnregisterImpl(obj, impl::GetTrueObjectContainerList(m_dummy_object_list)); + } + + template + ALWAYS_INLINE size_t KAutoObjectWithListContainer::GetOwnedCountChecked(const KProcess *owner) { + return this->GetOwnedCountImpl(owner, impl::GetTrueObjectContainerList(m_dummy_object_list)); + } + + inline u64 KAutoObjectWithList::GetId() const { + #define CLASS_TOKEN_HANDLER(CLASSNAME) \ + if constexpr (impl::IsAutoObjectWithSpecializedGetId) { \ + if (const CLASSNAME * const derived = this->DynamicCast(); derived != nullptr) { \ + return [](const T * const t_derived) ALWAYS_INLINE_LAMBDA -> u64 { \ + static_assert(std::same_as); \ + if constexpr (impl::IsAutoObjectWithSpecializedGetId) { \ + return impl::AutoObjectWithListComparator::GetRedBlackKey(*t_derived); \ + } else { \ + AMS_ASSUME(false); \ + } \ + }(derived); \ + } \ + } + + FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER) + #undef CLASS_TOKEN_HANDLER + + return impl::AutoObjectWithListComparator::GetRedBlackKey(*this); + } + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp index ad2ac1a68..a75b0cb55 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp @@ -21,6 +21,35 @@ namespace ams::kern { class KAutoObject; + #define FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(HANDLER) \ + HANDLER(KAutoObject) \ + \ + HANDLER(KSynchronizationObject) \ + HANDLER(KReadableEvent) \ + \ + HANDLER(KInterruptEvent) \ + HANDLER(KDebug) \ + HANDLER(KThread) \ + HANDLER(KServerPort) \ + HANDLER(KServerSession) \ + HANDLER(KClientPort) \ + HANDLER(KClientSession) \ + HANDLER(KProcess) \ + HANDLER(KResourceLimit) \ + HANDLER(KLightSession) \ + HANDLER(KPort) \ + HANDLER(KSession) \ + HANDLER(KSharedMemory) \ + HANDLER(KEvent) \ + HANDLER(KLightClientSession) \ + HANDLER(KLightServerSession) \ + HANDLER(KTransferMemory) \ + HANDLER(KDeviceAddressSpace) \ + HANDLER(KSessionRequest) \ + HANDLER(KCodeMemory) \ + HANDLER(KIoPool) \ + HANDLER(KIoRegion) + class KClassTokenGenerator { public: using TokenBaseType = u16; @@ -113,8 +142,11 @@ namespace ams::kern { KIoPool, KIoRegion, + FinalClassesLast, + FinalClassesEnd = FinalClassesStart + NumFinalClasses, }; + static_assert(ObjectType::FinalClassesLast <= ObjectType::FinalClassesEnd); template static constexpr inline TokenBaseType ClassToken = GetClassToken(); @@ -125,4 +157,37 @@ namespace ams::kern { template static constexpr inline ClassTokenType ClassToken = KClassTokenGenerator::ClassToken; + namespace impl { + + consteval bool IsKClassTokenGeneratorForEachMacroValid() { + auto IsObjectTypeIncludedByMacro = [](KClassTokenGenerator::ObjectType object_type) -> bool { + #define CLASS_TOKEN_HANDLER(CLASSNAME) if (object_type == KClassTokenGenerator::ObjectType::CLASSNAME) { return true; } + FOR_EACH_K_CLASS_TOKEN_OBJECT_TYPE(CLASS_TOKEN_HANDLER) + #undef CLASS_TOKEN_HANDLER + return false; + }; + + if (!IsObjectTypeIncludedByMacro(KClassTokenGenerator::ObjectType::KAutoObject)) { + return false; + } + + for (auto base = util::ToUnderlying(KClassTokenGenerator::ObjectType::BaseClassesStart); base < util::ToUnderlying(KClassTokenGenerator::ObjectType::BaseClassesEnd); ++base) { + if (!IsObjectTypeIncludedByMacro(static_cast(base))) { + return false; + } + } + + for (auto fin = util::ToUnderlying(KClassTokenGenerator::ObjectType::FinalClassesStart); fin < util::ToUnderlying(KClassTokenGenerator::ObjectType::FinalClassesLast); ++fin) { + if (!IsObjectTypeIncludedByMacro(static_cast(fin))) { + return false; + } + } + + return true; + } + + static_assert(IsKClassTokenGeneratorForEachMacroValid()); + + } + } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp index 52fac88c0..9680415eb 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -390,7 +390,8 @@ namespace ams::kern { void Finalize(); - virtual u64 GetId() const override final { return this->GetProcessId(); } + ALWAYS_INLINE u64 GetIdImpl() const { return this->GetProcessId(); } + ALWAYS_INLINE u64 GetId() const { return this->GetIdImpl(); } virtual bool IsSignaled() const override { MESOSPHERE_ASSERT_THIS(); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp index f7011ca93..930017f00 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -610,7 +610,8 @@ namespace ams::kern { size_t GetKernelStackUsage() const; public: /* Overridden parent functions. */ - virtual u64 GetId() const override final { return this->GetThreadId(); } + ALWAYS_INLINE u64 GetIdImpl() const { return this->GetThreadId(); } + ALWAYS_INLINE u64 GetId() const { return this->GetIdImpl(); } bool IsInitialized() const { return m_initialized; } uintptr_t GetPostDestroyArgument() const { return reinterpret_cast(m_parent) | (m_resource_limit_release_hint ? 1 : 0); } @@ -664,16 +665,6 @@ namespace ams::kern { return GetCurrentThread().GetCurrentCore(); } - ALWAYS_INLINE void KAutoObject::ScheduleDestruction() { - MESOSPHERE_ASSERT_THIS(); - - /* Set our object to destroy. */ - m_next_closed_object = GetCurrentThread().GetClosedObject(); - - /* Set ourselves as the thread's next object to destroy. */ - GetCurrentThread().SetClosedObject(this); - } - ALWAYS_INLINE void KTimerTask::OnTimer() { static_cast(this)->OnTimer(); } diff --git a/libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp b/libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp index fd7fc8fc3..69a6a44fc 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp @@ -68,7 +68,7 @@ namespace ams::kern { class KAutoObjectWithSlabHeapAndContainer : public Base { private: static constinit inline KSlabHeap s_slab_heap; - static constinit inline KAutoObjectWithListContainer s_container; + static constinit inline KAutoObjectWithListContainer s_container; private: static ALWAYS_INLINE Derived *Allocate() { return s_slab_heap.Allocate(); @@ -78,9 +78,9 @@ namespace ams::kern { s_slab_heap.Free(obj); } public: - class ListAccessor : public KAutoObjectWithListContainer::ListAccessor { + class ListAccessor : public KAutoObjectWithListContainer::ListAccessor { public: - ALWAYS_INLINE ListAccessor() : KAutoObjectWithListContainer::ListAccessor(s_container) { /* ... */ } + ALWAYS_INLINE ListAccessor() : KAutoObjectWithListContainer::ListAccessor(s_container) { /* ... */ } ALWAYS_INLINE ~ListAccessor() { /* ... */ } }; private: @@ -111,7 +111,7 @@ namespace ams::kern { Derived * const derived = static_cast(this); if (IsInitialized(derived)) { - s_container.Unregister(this); + s_container.Unregister(derived); const uintptr_t arg = GetPostDestroyArgument(derived); derived->Finalize(); Free(derived);