From 5e0bbb61b130b7bc35bedaf234ca8d3cdacd322f Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Tue, 28 Sep 2021 17:01:11 -0700 Subject: [PATCH] os: implement ReadWriteBusyMutex --- .../include/stratosphere/os.hpp | 2 +- .../os/impl/os_internal_rw_busy_mutex.hpp | 43 +++ ...internal_rw_busy_mutex_impl.os.horizon.hpp | 37 +++ .../stratosphere/os/os_rw_busy_mutex.hpp | 76 +++++ .../stratosphere/os/os_rw_busy_mutex_api.hpp | 32 ++ .../os/os_rw_busy_mutex_types.hpp | 31 ++ ...internal_rw_busy_mutex_impl.os.horizon.hpp | 277 ++++++++++++++++++ .../source/os/os_busy_mutex.cpp | 1 - .../source/os/os_rw_busy_mutex.cpp | 52 ++++ 9 files changed, 549 insertions(+), 2 deletions(-) create mode 100644 libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_api.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_types.hpp create mode 100644 libraries/libstratosphere/source/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp create mode 100644 libraries/libstratosphere/source/os/os_rw_busy_mutex.cpp diff --git a/libraries/libstratosphere/include/stratosphere/os.hpp b/libraries/libstratosphere/include/stratosphere/os.hpp index e1b29c6c1..d23174b52 100644 --- a/libraries/libstratosphere/include/stratosphere/os.hpp +++ b/libraries/libstratosphere/include/stratosphere/os.hpp @@ -31,7 +31,7 @@ #include #include #include -//#include +#include #include #include #include diff --git a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex.hpp b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex.hpp new file mode 100644 index 000000000..a8aa7d0b7 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include + +#if defined(ATMOSPHERE_OS_HORIZON) + #include +#else + #error "Unknown OS for ams::os::impl::InternalReadWriteBusyMutexImpl" +#endif + +namespace ams::os::impl { + + class InternalReadWriteBusyMutex { + private: + InternalReadWriteBusyMutexImpl m_impl; + public: + constexpr InternalReadWriteBusyMutex() : m_impl() { /* ... */ } + + ALWAYS_INLINE void AcquireReadLock() { return m_impl.AcquireReadLock(); } + ALWAYS_INLINE void ReleaseReadLock() { return m_impl.ReleaseReadLock(); } + + ALWAYS_INLINE void AcquireWriteLock() { return m_impl.AcquireWriteLock(); } + ALWAYS_INLINE void ReleaseWriteLock() { return m_impl.ReleaseWriteLock(); } + }; + + using InternalReadWriteBusyMutexStorage = util::TypedStorage; + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp new file mode 100644 index 000000000..da58679ab --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include + +namespace ams::os::impl { + + class InternalReadWriteBusyMutexImpl { + private: + u32 m_value; + public: + constexpr InternalReadWriteBusyMutexImpl() : m_value(0) { /* ... */ } + + constexpr void Initialize() { m_value = 0; } + + void AcquireReadLock(); + void ReleaseReadLock(); + + void AcquireWriteLock(); + void ReleaseWriteLock(); + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex.hpp b/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex.hpp new file mode 100644 index 000000000..b6acef848 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex.hpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +namespace ams::os { + + class ReadWriteBusyMutex { + NON_COPYABLE(ReadWriteBusyMutex); + NON_MOVEABLE(ReadWriteBusyMutex); + private: + ReadWriteBusyMutexType m_rw_mutex; + public: + constexpr explicit ReadWriteBusyMutex() : m_rw_mutex{{0}} { /* ... */ } + + void AcquireReadLock() { + return os::AcquireReadLockBusyMutex(std::addressof(m_rw_mutex)); + } + + void ReleaseReadLock() { + return os::ReleaseReadLockBusyMutex(std::addressof(m_rw_mutex)); + } + + void AcquireWriteLock() { + return os::AcquireWriteLockBusyMutex(std::addressof(m_rw_mutex)); + } + + void ReleaseWriteLock() { + return os::ReleaseWriteLockBusyMutex(std::addressof(m_rw_mutex)); + } + + void lock_shared() { + return this->AcquireReadLock(); + } + + void unlock_shared() { + return this->ReleaseReadLock(); + } + + void lock() { + return this->AcquireWriteLock(); + } + + void unlock() { + return this->ReleaseWriteLock(); + } + + operator ReadWriteBusyMutexType &() { + return m_rw_mutex; + } + + operator const ReadWriteBusyMutexType &() const { + return m_rw_mutex; + } + + ReadWriteBusyMutexType *GetBase() { + return std::addressof(m_rw_mutex); + } + }; + +} \ No newline at end of file diff --git a/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_api.hpp b/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_api.hpp new file mode 100644 index 000000000..df4b62e6f --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_api.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include + +namespace ams::os { + + struct ReadWriteBusyMutexType; + + void InitalizeReadWriteLockBusyMutex(ReadWriteBusyMutexType *rw_mutex); + + void AcquireReadLockBusyMutex(ReadWriteBusyMutexType *rw_mutex); + void ReleaseReadLockBusyMutex(ReadWriteBusyMutexType *rw_mutex); + + void AcquireWriteLockBusyMutex(ReadWriteBusyMutexType *rw_mutex); + void ReleaseWriteLockBusyMutex(ReadWriteBusyMutexType *rw_mutex); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_types.hpp b/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_types.hpp new file mode 100644 index 000000000..f0f95179f --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_rw_busy_mutex_types.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +namespace ams::os { + + struct ReadWriteBusyMutexType { + union { + s32 _arr[sizeof(impl::InternalReadWriteBusyMutexStorage) / sizeof(s32)]; + impl::InternalReadWriteBusyMutexStorage _storage; + }; + }; + static_assert(std::is_trivial::value); + +} diff --git a/libraries/libstratosphere/source/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp b/libraries/libstratosphere/source/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp new file mode 100644 index 000000000..ffb46a340 --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::os::impl { + + namespace { + + constexpr inline u8 WriterCountMax = std::numeric_limits::max(); + + ALWAYS_INLINE u16 GetReaderCount(u32 v) { + return static_cast(v >> 0); + } + + ALWAYS_INLINE u8 GetWriterCurrent(u32 v) { + return static_cast(v >> 16); + } + + ALWAYS_INLINE u8 GetWriterNext(u32 v) { + return static_cast(v >> 24); + } + + ALWAYS_INLINE u32 IncrementWriterNext(u32 v) { + return v + (1u << 24); + } + + ALWAYS_INLINE bool IsWriteLocked(u32 v) { + return GetWriterCurrent(v) != GetWriterNext(v); + } + + ALWAYS_INLINE void PrefetchForBusyMutex(u32 *p) { + /* Nintendo does PRFM pstl1keep. */ + __builtin_prefetch(p, 1); + } + + ALWAYS_INLINE void SendEventLocalForBusyMutex() { + __asm__ __volatile__("sevl" ::: "memory"); + } + + ALWAYS_INLINE void WaitForEventsForBusyMutex() { + __asm__ __volatile__("wfe" ::: "memory"); + } + + ALWAYS_INLINE u32 LoadAcquireExclusiveForBusyMutex(u32 *p) { + u32 v; + __asm__ __volatile__("ldaxr %w[v], %[p]" : [v]"=&r"(v) : [p]"Q"(*p) : "memory"); + return v; + } + + ALWAYS_INLINE u32 LoadExclusiveForBusyMutex(u32 *p) { + u32 v; + __asm__ __volatile__("ldxr %w[v], %[p]" : [v]"=&r"(v) : [p]"Q"(*p) : "memory"); + return v; + } + + ALWAYS_INLINE bool StoreReleaseExclusiveForBusyMutex(u32 *p, u32 v) { + int result; + __asm__ __volatile__("stlxr %w[result], %w[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory"); + return result == 0; + } + + ALWAYS_INLINE bool StoreExclusiveForBusyMutex(u32 *p, u32 v) { + int result; + __asm__ __volatile__("stxr %w[result], %w[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory"); + return result == 0; + } + + ALWAYS_INLINE u8 *GetWriterCurrentPointerForBusyMutex(u32 *p) { + if constexpr (util::IsLittleEndian()) { + return reinterpret_cast(reinterpret_cast(p)) + 2; + } else { + return reinterpret_cast(reinterpret_cast(p)) + 1; + } + } + + ALWAYS_INLINE void StoreReleaseWriteLockValueForBusyMutex(u32 *p) { + u8 * const p8 = GetWriterCurrentPointerForBusyMutex(p); + + u8 v; + __asm__ __volatile__("ldrb %w[v], %[p8]\n" + "add %w[v], %w[v], #1\n" + "stlrb %w[v], %[p8]\n" + : [v]"=&r"(v) + : [p8]"Q"(*p8) + : "memory"); + } + + } + + void InternalReadWriteBusyMutexImpl::AcquireReadLock() { + /* Get the thread local region. */ + auto * const tlr = svc::GetThreadLocalRegion(); + + /* Determine disable counters. */ + const auto cur_dc = tlr->disable_count; + AMS_ABORT_UNLESS(cur_dc < std::numeric_limits::max()); + const auto next_dc = cur_dc + 1; + + /* Get pointer to our value. */ + u32 * const p = std::addressof(m_value); + + /* Pre-fetch the busy mutex. */ + PrefetchForBusyMutex(p); + + /* Acquire the read-lock for the mutex. */ + while (true) { + /* Set the updated disable counter. */ + tlr->disable_count = next_dc; + + /* Try to acquire. */ + const u32 v = LoadAcquireExclusiveForBusyMutex(p); + + /* We can only acquire read lock if not write-locked. */ + const bool write_locked = IsWriteLocked(v); + if (AMS_LIKELY(!write_locked)) { + /* Check that we don't overflow the reader count. */ + const u32 new_v = v + 1; + AMS_ABORT_UNLESS(GetReaderCount(new_v) != 0); + + /* Try to store our updated lock value. */ + if (AMS_LIKELY(StoreExclusiveForBusyMutex(p, new_v))) { + break; + } + } + + /* Reset the disable counter, since we failed to acquire. */ + tlr->disable_count = cur_dc; + + /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we tried to acquire the lock. */ + if (cur_dc == 0 && tlr->interrupt_flag) { + svc::SynchronizePreemptionState(); + } + + /* If the lock is held by another core, wait for it to be released. */ + if (write_locked) { + WaitForEventsForBusyMutex(); + } + } + } + + void InternalReadWriteBusyMutexImpl::ReleaseReadLock() { + /* Release the read lock. */ + { + /* Get pointer to our value. */ + u32 * const p = std::addressof(m_value); + + u32 v; + do { + /* Get and validate the current value. */ + v = LoadExclusiveForBusyMutex(p); + AMS_ABORT_UNLESS(GetReaderCount(v) != 0); + } while (!StoreReleaseExclusiveForBusyMutex(p, v - 1)); + } + + /* Get the thread local region. */ + auto * const tlr = svc::GetThreadLocalRegion(); + + /* Determine disable counters. */ + const auto cur_dc = tlr->disable_count; + AMS_ASSERT(cur_dc != 0); + const auto next_dc = cur_dc - 1; + + /* Decrement disable count. */ + tlr->disable_count = next_dc; + + /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we held the lock. */ + if (next_dc == 0 && tlr->interrupt_flag) { + svc::SynchronizePreemptionState(); + } + } + + void InternalReadWriteBusyMutexImpl::AcquireWriteLock() { + /* Get the thread local region. */ + auto * const tlr = svc::GetThreadLocalRegion(); + + /* Determine disable counters. */ + const auto cur_dc = tlr->disable_count; + AMS_ABORT_UNLESS(cur_dc < std::numeric_limits::max()); + const auto next_dc = cur_dc + 1; + + /* Get pointer to our value. */ + u32 * const p = std::addressof(m_value); + + /* Pre-fetch the busy mutex. */ + PrefetchForBusyMutex(p); + + /* Acquire the read-lock for the mutex. */ + while (true) { + /* Set the updated disable counter. */ + tlr->disable_count = next_dc; + + /* Try to acquire. */ + const u32 v = LoadAcquireExclusiveForBusyMutex(p); + + /* Check that we can write lock. */ + AMS_ABORT_UNLESS(static_cast(GetWriterNext(v) - GetWriterCurrent(v)) < WriterCountMax); + + /* Determine our write-lock number. */ + const u32 new_v = IncrementWriterNext(v); + + /* Try to store our updated lock value. */ + if (AMS_UNLIKELY(!StoreExclusiveForBusyMutex(p, new_v))) { + /* Reset the disable counter, since we failed to acquire. */ + tlr->disable_count = cur_dc; + + /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we tried to acquire the lock. */ + if (cur_dc == 0 && tlr->interrupt_flag) { + svc::SynchronizePreemptionState(); + } + + continue; + } + + /* Wait until the lock is truly acquired. */ + if (GetReaderCount(new_v) != 0 || GetWriterNext(v) != GetWriterCurrent(new_v)) { + /* Send an event, so that we can immediately wait without fail. */ + SendEventLocalForBusyMutex(); + + while (true) { + /* Wait for a lock update. */ + WaitForEventsForBusyMutex(); + + /* Get the updated value. */ + const u32 cur_v = LoadAcquireExclusiveForBusyMutex(p); + if (GetReaderCount(cur_v) == 0 && GetWriterNext(v) == GetWriterCurrent(cur_v)) { + break; + } + } + } + + /* We've acquired the write lock. */ + break; + } + } + + void InternalReadWriteBusyMutexImpl::ReleaseWriteLock() { + /* Check pre-conditions. */ + AMS_ABORT_UNLESS(IsWriteLocked(m_value)); + + /* Get pointer to our value. */ + u32 * const p = std::addressof(m_value); + + /* Release the write lock. */ + StoreReleaseWriteLockValueForBusyMutex(p); + + /* Get the thread local region. */ + auto * const tlr = svc::GetThreadLocalRegion(); + + /* Determine disable counters. */ + const auto cur_dc = tlr->disable_count; + AMS_ASSERT(cur_dc != 0); + const auto next_dc = cur_dc - 1; + + /* Decrement disable count. */ + tlr->disable_count = next_dc; + + /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we held the lock. */ + if (next_dc == 0 && tlr->interrupt_flag) { + svc::SynchronizePreemptionState(); + } + } + +} diff --git a/libraries/libstratosphere/source/os/os_busy_mutex.cpp b/libraries/libstratosphere/source/os/os_busy_mutex.cpp index 379044559..309833ea0 100644 --- a/libraries/libstratosphere/source/os/os_busy_mutex.cpp +++ b/libraries/libstratosphere/source/os/os_busy_mutex.cpp @@ -24,7 +24,6 @@ namespace ams::os { - void InitializeBusyMutex(BusyMutexType *mutex) { /* Create object. */ util::ConstructAt(mutex->_storage); diff --git a/libraries/libstratosphere/source/os/os_rw_busy_mutex.cpp b/libraries/libstratosphere/source/os/os_rw_busy_mutex.cpp new file mode 100644 index 000000000..aa2a94b72 --- /dev/null +++ b/libraries/libstratosphere/source/os/os_rw_busy_mutex.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "impl/os_thread_manager.hpp" + +#if defined(ATMOSPHERE_OS_HORIZON) + #include "impl/os_internal_rw_busy_mutex_impl.os.horizon.hpp" +#else + #error "Unknown OS for ams::os::impl::InternalReadWriteBusyMutexImpl" +#endif + +namespace ams::os { + + void InitalizeReadWriteLockBusyMutex(ReadWriteBusyMutexType *rw_mutex) { + /* Create object. */ + util::ConstructAt(rw_mutex->_storage); + } + + void AcquireReadLockBusyMutex(ReadWriteBusyMutexType *rw_mutex) { + /* Acquire read lock. */ + util::GetReference(rw_mutex->_storage).AcquireReadLock(); + } + + void ReleaseReadLockBusyMutex(ReadWriteBusyMutexType *rw_mutex) { + /* Release read lock. */ + util::GetReference(rw_mutex->_storage).ReleaseReadLock(); + } + + void AcquireWriteLockBusyMutex(ReadWriteBusyMutexType *rw_mutex) { + /* Acquire write lock. */ + util::GetReference(rw_mutex->_storage).AcquireWriteLock(); + } + + void ReleaseWriteLockBusyMutex(ReadWriteBusyMutexType *rw_mutex) { + /* Release write lock. */ + util::GetReference(rw_mutex->_storage).ReleaseWriteLock(); + } + +}