os: implement BusyMutex

libraries/libstratosphere/include/stratosphere/os.hpp

@@ -30,6 +30,8 @@
 #include <stratosphere/os/os_condition_variable.hpp>
 #include <stratosphere/os/os_sdk_mutex.hpp>
 #include <stratosphere/os/os_sdk_condition_variable.hpp>
+#include <stratosphere/os/os_busy_mutex.hpp>
+//#include <stratosphere/os/os_rw_busy_mutex.hpp>
 #include <stratosphere/os/os_rw_lock.hpp>
 #include <stratosphere/os/os_transfer_memory.hpp>
 #include <stratosphere/os/os_semaphore.hpp>
@@ -42,4 +44,7 @@
 #include <stratosphere/os/os_sdk_reply_and_receive.hpp>
 #include <stratosphere/os/os_thread.hpp>
 #include <stratosphere/os/os_message_queue.hpp>
+//#include <stratosphere/os/os_light_event.hpp>
+//#include <stratosphere/os/os_light_message_queue.hpp>
+//#include <stratosphere/os/os_light_semaphore.hpp>
 #include <stratosphere/os/os_waitable.hpp>

libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex.hpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <vapours.hpp>
+
+#if defined(ATMOSPHERE_OS_HORIZON)
+    #include <stratosphere/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp>
+#else
+    #error "Unknown OS for ams::os::impl::InternalBusyMutexImpl"
+#endif
+
+namespace ams::os::impl {
+
+    class InternalBusyMutex {
+        private:
+            InternalBusyMutexImpl m_impl;
+        public:
+            constexpr InternalBusyMutex() : m_impl() { /* ... */ }
+
+            constexpr void Initialize() { m_impl.Initialize(); }
+            constexpr void Finalize()   { m_impl.Finalize(); }
+
+            bool IsLocked() const { return m_impl.IsLocked(); }
+
+            ALWAYS_INLINE void Lock()    { return m_impl.Lock(); }
+            ALWAYS_INLINE bool TryLock() { return m_impl.TryLock(); }
+            ALWAYS_INLINE void Unlock()  { return m_impl.Unlock(); }
+
+            ALWAYS_INLINE void lock()     { return this->Lock(); }
+            ALWAYS_INLINE bool try_lock() { return this->TryLock(); }
+            ALWAYS_INLINE void unlock()   { return this->Unlock(); }
+    };
+
+    using InternalBusyMutexStorage = util::TypedStorage<InternalBusyMutex>;
+
+}

libraries/libstratosphere/include/stratosphere/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <vapours.hpp>
+
+namespace ams::os::impl {
+
+    class InternalBusyMutexImpl {
+        private:
+            u32 m_value;
+        public:
+            constexpr InternalBusyMutexImpl() : m_value(0) { /* ... */ }
+
+            constexpr void Initialize() { m_value = 0; }
+            constexpr void Finalize() { /* ... */ }
+
+            constexpr bool IsLocked() const { return m_value != 0; }
+
+            void Lock();
+            bool TryLock();
+            void Unlock();
+    };
+
+}

libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp

@@ -38,7 +38,7 @@ namespace ams::os::impl {
             constexpr InternalCriticalSectionImpl() : thread_handle(svc::InvalidHandle) { /* ... */ }
 
             constexpr void Initialize() { this->thread_handle = svc::InvalidHandle; }
-            constexpr void Finalize() { /* ... */}
+            constexpr void Finalize() { /* ... */ }
 
             void Enter();
             bool TryEnter();

libraries/libstratosphere/include/stratosphere/os/os_busy_mutex.hpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <stratosphere/os/os_busy_mutex_types.hpp>
+#include <stratosphere/os/os_busy_mutex_api.hpp>
+
+namespace ams::os {
+
+    class BusyMutex {
+        NON_COPYABLE(BusyMutex);
+        NON_MOVEABLE(BusyMutex);
+        private:
+            BusyMutexType m_mutex;
+        public:
+            constexpr explicit BusyMutex() : m_mutex{::ams::os::BusyMutexType::State_Initialized, nullptr, {{}}} { /* ... */ }
+
+            ~BusyMutex() { FinalizeBusyMutex(std::addressof(m_mutex)); }
+
+            void lock() {
+                return LockBusyMutex(std::addressof(m_mutex));
+            }
+
+            void unlock() {
+                return UnlockBusyMutex(std::addressof(m_mutex));
+            }
+
+            bool try_lock() {
+                return TryLockBusyMutex(std::addressof(m_mutex));
+            }
+
+            ALWAYS_INLINE void Lock() {
+                return this->lock();
+            }
+
+            ALWAYS_INLINE void Unlock() {
+                return this->unlock();
+            }
+
+            ALWAYS_INLINE bool TryLock() {
+                return this->try_lock();
+            }
+
+            operator BusyMutexType &() {
+                return m_mutex;
+            }
+
+            operator const BusyMutexType &() const {
+                return m_mutex;
+            }
+
+            BusyMutexType *GetBase() {
+                return std::addressof(m_mutex);
+            }
+    };
+
+}

libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_api.hpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <vapours.hpp>
+
+namespace ams::os {
+
+    struct BusyMutexType;
+
+    void InitializeBusyMutex(BusyMutexType *mutex);
+    void FinalizeBusyMutex(BusyMutexType *mutex);
+
+    void LockBusyMutex(BusyMutexType *mutex);
+    bool TryLockBusyMutex(BusyMutexType *mutex);
+    void UnlockBusyMutex(BusyMutexType *mutex);
+
+}

libraries/libstratosphere/include/stratosphere/os/os_busy_mutex_types.hpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <vapours.hpp>
+#include <stratosphere/os/impl/os_internal_busy_mutex.hpp>
+
+namespace ams::os {
+
+    struct ThreadType;
+
+    struct BusyMutexType {
+        enum State {
+            State_NotInitialized = 0,
+            State_Initialized    = 1,
+        };
+
+        u8 state;
+        ThreadType *owner_thread;
+        union {
+            s32 _arr[sizeof(impl::InternalBusyMutexStorage) / sizeof(s32)];
+            impl::InternalBusyMutexStorage _storage;
+        };
+    };
+    static_assert(std::is_trivial<BusyMutexType>::value);
+
+}

libraries/libstratosphere/source/os/impl/os_internal_busy_mutex_impl.os.horizon.hpp

@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <stratosphere.hpp>
+
+namespace ams::os::impl {
+
+    namespace {
+
+        ALWAYS_INLINE void PrefetchForBusyMutex(u32 *p) {
+            /* Nintendo does PRFM pstl1keep. */
+            __builtin_prefetch(p, 1);
+        }
+
+        ALWAYS_INLINE void WaitForEventsForBusyMutex() {
+            __asm__ __volatile__("wfe" ::: "memory");
+        }
+
+        ALWAYS_INLINE u32 LoadExclusiveForBusyMutex(u32 *p) {
+            u32 v;
+            __asm__ __volatile__("ldaxr %w[v], %[p]" : [v]"=&r"(v) : [p]"Q"(*p) : "memory");
+            return v;
+        }
+
+        ALWAYS_INLINE bool StoreExclusiveForBusyMutex(u32 *p, u32 v) {
+            int result;
+            __asm__ __volatile__("stxr %w[result], %w[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory");
+            return result == 0;
+        }
+
+        ALWAYS_INLINE void ClearExclusiveForBusyMutex() {
+            __asm__ __volatile__("clrex" ::: "memory");
+        }
+
+        ALWAYS_INLINE void StoreUnlockValueForBusyMutex(u32 *p) {
+            __asm__ __volatile__("stlr wzr, %[p]" :: [p]"Q"(*p) : "memory");
+        }
+
+    }
+
+    void InternalBusyMutexImpl::Lock() {
+        /* Get the thread local region. */
+        auto * const tlr = svc::GetThreadLocalRegion();
+
+        /* Determine disable counters. */
+        const auto cur_dc = tlr->disable_count;
+        AMS_ASSERT(cur_dc < std::numeric_limits<decltype(cur_dc)>::max());
+        const auto next_dc = cur_dc + 1;
+
+        /* Get pointer to our value. */
+        u32 * const p = std::addressof(m_value);
+
+        /* Pre-fetch the busy mutex. */
+        PrefetchForBusyMutex(p);
+
+        /* Acquire the busy mutex. */
+        while (true) {
+            /* Set the updated disable counter. */
+            tlr->disable_count = next_dc;
+
+            /* Try to acquire. */
+            const u32 v = LoadExclusiveForBusyMutex(p);
+            if (AMS_LIKELY(v == 0) && AMS_LIKELY(StoreExclusiveForBusyMutex(p, 1))) {
+                break;
+            }
+
+            /* Reset the disable counter, since we failed to acquire. */
+            tlr->disable_count = cur_dc;
+
+            /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we tried to acquire the lock. */
+            if (cur_dc == 0 && tlr->interrupt_flag) {
+                svc::SynchronizePreemptionState();
+            }
+
+            /* If the lock is held by another core, wait for it to be released. */
+            if (v != 0) {
+                WaitForEventsForBusyMutex();
+            }
+        }
+    }
+
+    bool InternalBusyMutexImpl::TryLock() {
+        /* Get the thread local region. */
+        auto * const tlr = svc::GetThreadLocalRegion();
+
+        /* Determine disable counters. */
+        const auto cur_dc = tlr->disable_count;
+        AMS_ASSERT(cur_dc < std::numeric_limits<decltype(cur_dc)>::max());
+        const auto next_dc = cur_dc + 1;
+
+        /* Get pointer to our value. */
+        u32 * const p = std::addressof(m_value);
+
+        /* Pre-fetch the busy mutex. */
+        PrefetchForBusyMutex(p);
+
+        /* Try to acquire the busy mutex. */
+        while (true) {
+            /* Set the updated disable counter. */
+            tlr->disable_count = next_dc;
+
+            /* Ensure we do whatever cleanup we need to. */
+            auto release_guard = SCOPE_GUARD {
+                /* Reset disable counter. */
+                tlr->disable_count = cur_dc;
+
+                /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we tried to acquire the lock. */
+                if (cur_dc == 0 && tlr->interrupt_flag) {
+                    svc::SynchronizePreemptionState();
+                }
+            };
+
+            /* Try to acquire. */
+            const u32 v = LoadExclusiveForBusyMutex(p);
+            if (AMS_UNLIKELY(v != 0)) {
+                ClearExclusiveForBusyMutex();
+                return false;
+            }
+
+            if (AMS_LIKELY(StoreExclusiveForBusyMutex(p, 1))) {
+                /* We successfully acquired the busy mutex. */
+                release_guard.Cancel();
+                return true;
+            }
+        }
+    }
+
+    void InternalBusyMutexImpl::Unlock() {
+        /* Get pointer to our value. */
+        u32 * const p = std::addressof(m_value);
+
+        /* Unlock the mutex. */
+        StoreUnlockValueForBusyMutex(p);
+
+        /* Get the thread local region. */
+        auto * const tlr = svc::GetThreadLocalRegion();
+
+        /* Determine disable counters. */
+        const auto cur_dc = tlr->disable_count;
+        AMS_ASSERT(cur_dc != 0);
+        const auto next_dc = cur_dc - 1;
+
+        /* Decrement disable count. */
+        tlr->disable_count = next_dc;
+
+        /* If we don't hold any other busy mutexes, acknowledge any interrupts that occurred while we held the lock. */
+        if (next_dc == 0 && tlr->interrupt_flag) {
+            svc::SynchronizePreemptionState();
+        }
+    }
+
+}

libraries/libstratosphere/source/os/os_busy_mutex.cpp

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <stratosphere.hpp>
+#include "impl/os_thread_manager.hpp"
+
+#if defined(ATMOSPHERE_OS_HORIZON)
+    #include "impl/os_internal_busy_mutex_impl.os.horizon.hpp"
+#else
+    #error "Unknown OS for ams::os::impl::InternalBusyMutexImpl"
+#endif
+
+namespace ams::os {
+
+
+    void InitializeBusyMutex(BusyMutexType *mutex) {
+        /* Create object. */
+        util::ConstructAt(mutex->_storage);
+
+        /* Set member variables. */
+        mutex->owner_thread = nullptr;
+
+        /* Mark initialized. */
+        mutex->state = BusyMutexType::State_Initialized;
+    }
+
+    void FinalizeBusyMutex(BusyMutexType *mutex) {
+        /* Check pre-conditions. */
+        AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized);
+        AMS_ASSERT(!util::GetReference(mutex->_storage).IsLocked());
+
+        /* Mark not intialized. */
+        mutex->state = MutexType::State_NotInitialized;
+
+        /* Destroy object. */
+        util::DestroyAt(mutex->_storage);
+    }
+
+    void LockBusyMutex(BusyMutexType *mutex) {
+        /* Check pre-conditions. */
+        AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized);
+
+        /* Lock mutex. */
+        util::GetReference(mutex->_storage).Lock();
+
+        /* Set owner thread. */
+        mutex->owner_thread = impl::GetCurrentThread();
+    }
+
+    bool TryLockBusyMutex(BusyMutexType *mutex) {
+        /* Check pre-conditions. */
+        AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized);
+
+        /* Try to lock mutex. */
+        const bool locked = util::GetReference(mutex->_storage).TryLock();
+
+        /* Set owner thread. */
+        if (locked) {
+            mutex->owner_thread = impl::GetCurrentThread();
+        }
+
+        return locked;
+    }
+
+    void UnlockBusyMutex(BusyMutexType *mutex) {
+        /* Check pre-conditions. */
+        AMS_ASSERT(mutex->state == BusyMutexType::State_Initialized);
+        AMS_ASSERT(util::GetReference(mutex->_storage).IsLocked() && mutex->owner_thread == impl::GetCurrentThread());
+
+        /* Unlock. */
+        util::GetReference(mutex->_storage).Unlock();
+    }
+
+}