From 09570c470c7ffb95fc83b7bf83511b0098bfe43b Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Tue, 28 Sep 2021 14:53:55 -0700 Subject: [PATCH] os: implement BusyMutex --- .../include/stratosphere/os.hpp | 5 + .../os/impl/os_internal_busy_mutex.hpp | 50 ++++++ ...os_internal_busy_mutex_impl.os.horizon.hpp | 38 ++++ ...ernal_critical_section_impl.os.horizon.hpp | 2 +- .../include/stratosphere/os/os_busy_mutex.hpp | 70 ++++++++ .../stratosphere/os/os_busy_mutex_api.hpp | 31 ++++ .../stratosphere/os/os_busy_mutex_types.hpp | 40 +++++ ...os_internal_busy_mutex_impl.os.horizon.hpp | 165 ++++++++++++++++++ .../source/os/os_busy_mutex.cpp | 86 +++++++++ 9 files changed, 486 insertions(+), 1 deletion(-) create mode 100644 libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_busy_mutex.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_api.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_types.hpp create mode 100644 libraries/libstratosphere/source/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp create mode 100644 libraries/libstratosphere/source/os/os_busy_mutex.cpp diff --git a/libraries/libstratosphere/include/stratosphere/os.hpp b/libraries/libstratosphere/include/stratosphere/os.hpp index d2393ad7b..e1b29c6c1 100644 --- a/libraries/libstratosphere/include/stratosphere/os.hpp +++ b/libraries/libstratosphere/include/stratosphere/os.hpp @@ -30,6 +30,8 @@ #include #include #include +#include +//#include #include #include #include @@ -42,4 +44,7 @@ #include #include #include +//#include +//#include +//#include #include diff --git a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex.hpp b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex.hpp new file mode 100644 index 000000000..078474bc0 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include + +#if defined(ATMOSPHERE_OS_HORIZON) + #include +#else + #error "Unknown OS for ams::os::impl::InternalBusyMutexImpl" +#endif + +namespace ams::os::impl { + + class InternalBusyMutex { + private: + InternalBusyMutexImpl m_impl; + public: + constexpr InternalBusyMutex() : m_impl() { /* ... */ } + + constexpr void Initialize() { m_impl.Initialize(); } + constexpr void Finalize() { m_impl.Finalize(); } + + bool IsLocked() const { return m_impl.IsLocked(); } + + ALWAYS_INLINE void Lock() { return m_impl.Lock(); } + ALWAYS_INLINE bool TryLock() { return m_impl.TryLock(); } + ALWAYS_INLINE void Unlock() { return m_impl.Unlock(); } + + ALWAYS_INLINE void lock() { return this->Lock(); } + ALWAYS_INLINE bool try_lock() { return this->TryLock(); } + ALWAYS_INLINE void unlock() { return this->Unlock(); } + }; + + using InternalBusyMutexStorage = util::TypedStorage; + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp new file mode 100644 index 000000000..769ae0e19 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include + +namespace ams::os::impl { + + class InternalBusyMutexImpl { + private: + u32 m_value; + public: + constexpr InternalBusyMutexImpl() : m_value(0) { /* ... */ } + + constexpr void Initialize() { m_value = 0; } + constexpr void Finalize() { /* ... */ } + + constexpr bool IsLocked() const { return m_value != 0; } + + void Lock(); + bool TryLock(); + void Unlock(); + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp index b4c611cc3..76ea12f2e 100644 --- a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp +++ b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp @@ -38,7 +38,7 @@ namespace ams::os::impl { constexpr InternalCriticalSectionImpl() : thread_handle(svc::InvalidHandle) { /* ... */ } constexpr void Initialize() { this->thread_handle = svc::InvalidHandle; } - constexpr void Finalize() { /* ... */} + constexpr void Finalize() { /* ... */ } void Enter(); bool TryEnter(); diff --git a/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex.hpp b/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex.hpp new file mode 100644 index 000000000..500808c0a --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +namespace ams::os { + + class BusyMutex { + NON_COPYABLE(BusyMutex); + NON_MOVEABLE(BusyMutex); + private: + BusyMutexType m_mutex; + public: + constexpr explicit BusyMutex() : m_mutex{::ams::os::BusyMutexType::State_Initialized, nullptr, {{}}} { /* ... */ } + + ~BusyMutex() { FinalizeBusyMutex(std::addressof(m_mutex)); } + + void lock() { + return LockBusyMutex(std::addressof(m_mutex)); + } + + void unlock() { + return UnlockBusyMutex(std::addressof(m_mutex)); + } + + bool try_lock() { + return TryLockBusyMutex(std::addressof(m_mutex)); + } + + ALWAYS_INLINE void Lock() { + return this->lock(); + } + + ALWAYS_INLINE void Unlock() { + return this->unlock(); + } + + ALWAYS_INLINE bool TryLock() { + return this->try_lock(); + } + + operator BusyMutexType &() { + return m_mutex; + } + + operator const BusyMutexType &() const { + return m_mutex; + } + + BusyMutexType *GetBase() { + return std::addressof(m_mutex); + } + }; + +} \ No newline at end of file diff --git a/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_api.hpp b/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_api.hpp new file mode 100644 index 000000000..8371fadb3 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_api.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include + +namespace ams::os { + + struct BusyMutexType; + + void InitializeBusyMutex(BusyMutexType *mutex); + void FinalizeBusyMutex(BusyMutexType *mutex); + + void LockBusyMutex(BusyMutexType *mutex); + bool TryLockBusyMutex(BusyMutexType *mutex); + void UnlockBusyMutex(BusyMutexType *mutex); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_types.hpp b/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_types.hpp new file mode 100644 index 000000000..981e3ce11 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_types.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +namespace ams::os { + + struct ThreadType; + + struct BusyMutexType { + enum State { + State_NotInitialized = 0, + State_Initialized = 1, + }; + + u8 state; + ThreadType *owner_thread; + union { + s32 _arr[sizeof(impl::InternalBusyMutexStorage) / sizeof(s32)]; + impl::InternalBusyMutexStorage _storage; + }; + }; + static_assert(std::is_trivial::value); + +} diff --git a/libraries/libstratosphere/source/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp b/libraries/libstratosphere/source/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp new file mode 100644 index 000000000..f0c6fefa5 --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::os::impl { + + namespace { + + ALWAYS_INLINE void PrefetchForBusyMutex(u32 *p) { + /* Nintendo does PRFM pstl1keep. */ + __builtin_prefetch(p, 1); + } + + ALWAYS_INLINE void WaitForEventsForBusyMutex() { + __asm__ __volatile__("wfe" ::: "memory"); + } + + ALWAYS_INLINE u32 LoadExclusiveForBusyMutex(u32 *p) { + u32 v; + __asm__ __volatile__("ldaxr %w[v], %[p]" : [v]"=&r"(v) : [p]"Q"(*p) : "memory"); + return v; + } + + ALWAYS_INLINE bool StoreExclusiveForBusyMutex(u32 *p, u32 v) { + int result; + __asm__ __volatile__("stxr %w[result], %w[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory"); + return result == 0; + } + + ALWAYS_INLINE void ClearExclusiveForBusyMutex() { + __asm__ __volatile__("clrex" ::: "memory"); + } + + ALWAYS_INLINE void StoreUnlockValueForBusyMutex(u32 *p) { + __asm__ __volatile__("stlr wzr, %[p]" :: [p]"Q"(*p) : "memory"); + } + + } + + void InternalBusyMutexImpl::Lock() { + /* Get the thread local region. */ + auto * const tlr = svc::GetThreadLocalRegion(); + + /* Determine disable counters. */ + const auto cur_dc = tlr->disable_count; + AMS_ASSERT(cur_dc < std::numeric_limits::max()); + const auto next_dc = cur_dc + 1; + + /* Get pointer to our value. */ + u32 * const p = std::addressof(m_value); + + /* Pre-fetch the busy mutex. */ + PrefetchForBusyMutex(p); + + /* Acquire the busy mutex. */ + while (true) { + /* Set the updated disable counter. */ + tlr->disable_count = next_dc; + + /* Try to acquire. */ + const u32 v = LoadExclusiveForBusyMutex(p); + if (AMS_LIKELY(v == 0) && AMS_LIKELY(StoreExclusiveForBusyMutex(p, 1))) { + break; + } + + /* Reset the disable counter, since we failed to acquire. */ + tlr->disable_count = cur_dc; + + /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we tried to acquire the lock. */ + if (cur_dc == 0 && tlr->interrupt_flag) { + svc::SynchronizePreemptionState(); + } + + /* If the lock is held by another core, wait for it to be released. */ + if (v != 0) { + WaitForEventsForBusyMutex(); + } + } + } + + bool InternalBusyMutexImpl::TryLock() { + /* Get the thread local region. */ + auto * const tlr = svc::GetThreadLocalRegion(); + + /* Determine disable counters. */ + const auto cur_dc = tlr->disable_count; + AMS_ASSERT(cur_dc < std::numeric_limits::max()); + const auto next_dc = cur_dc + 1; + + /* Get pointer to our value. */ + u32 * const p = std::addressof(m_value); + + /* Pre-fetch the busy mutex. */ + PrefetchForBusyMutex(p); + + /* Try to acquire the busy mutex. */ + while (true) { + /* Set the updated disable counter. */ + tlr->disable_count = next_dc; + + /* Ensure we do whatever cleanup we need to. */ + auto release_guard = SCOPE_GUARD { + /* Reset disable counter. */ + tlr->disable_count = cur_dc; + + /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we tried to acquire the lock. */ + if (cur_dc == 0 && tlr->interrupt_flag) { + svc::SynchronizePreemptionState(); + } + }; + + /* Try to acquire. */ + const u32 v = LoadExclusiveForBusyMutex(p); + if (AMS_UNLIKELY(v != 0)) { + ClearExclusiveForBusyMutex(); + return false; + } + + if (AMS_LIKELY(StoreExclusiveForBusyMutex(p, 1))) { + /* We successfully acquired the busy mutex. */ + release_guard.Cancel(); + return true; + } + } + } + + void InternalBusyMutexImpl::Unlock() { + /* Get pointer to our value. */ + u32 * const p = std::addressof(m_value); + + /* Unlock the mutex. */ + StoreUnlockValueForBusyMutex(p); + + /* Get the thread local region. */ + auto * const tlr = svc::GetThreadLocalRegion(); + + /* Determine disable counters. */ + const auto cur_dc = tlr->disable_count; + AMS_ASSERT(cur_dc != 0); + const auto next_dc = cur_dc - 1; + + /* Decrement disable count. */ + tlr->disable_count = next_dc; + + /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we held the lock. */ + if (next_dc == 0 && tlr->interrupt_flag) { + svc::SynchronizePreemptionState(); + } + } + +} diff --git a/libraries/libstratosphere/source/os/os_busy_mutex.cpp b/libraries/libstratosphere/source/os/os_busy_mutex.cpp new file mode 100644 index 000000000..379044559 --- /dev/null +++ b/libraries/libstratosphere/source/os/os_busy_mutex.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "impl/os_thread_manager.hpp" + +#if defined(ATMOSPHERE_OS_HORIZON) + #include "impl/os_internal_busy_mutex_impl.os.horizon.hpp" +#else + #error "Unknown OS for ams::os::impl::InternalBusyMutexImpl" +#endif + +namespace ams::os { + + + void InitializeBusyMutex(BusyMutexType *mutex) { + /* Create object. */ + util::ConstructAt(mutex->_storage); + + /* Set member variables. */ + mutex->owner_thread = nullptr; + + /* Mark initialized. */ + mutex->state = BusyMutexType::State_Initialized; + } + + void FinalizeBusyMutex(BusyMutexType *mutex) { + /* Check pre-conditions. */ + AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized); + AMS_ASSERT(!util::GetReference(mutex->_storage).IsLocked()); + + /* Mark not intialized. */ + mutex->state = MutexType::State_NotInitialized; + + /* Destroy object. */ + util::DestroyAt(mutex->_storage); + } + + void LockBusyMutex(BusyMutexType *mutex) { + /* Check pre-conditions. */ + AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized); + + /* Lock mutex. */ + util::GetReference(mutex->_storage).Lock(); + + /* Set owner thread. */ + mutex->owner_thread = impl::GetCurrentThread(); + } + + bool TryLockBusyMutex(BusyMutexType *mutex) { + /* Check pre-conditions. */ + AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized); + + /* Try to lock mutex. */ + const bool locked = util::GetReference(mutex->_storage).TryLock(); + + /* Set owner thread. */ + if (locked) { + mutex->owner_thread = impl::GetCurrentThread(); + } + + return locked; + } + + void UnlockBusyMutex(BusyMutexType *mutex) { + /* Check pre-conditions. */ + AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized); + AMS_ASSERT(util::GetReference(mutex->_storage).IsLocked() && mutex->owner_thread == impl::GetCurrentThread()); + + /* Unlock. */ + util::GetReference(mutex->_storage).Unlock(); + } + +}