mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-01-30 12:53:52 +00:00
Code Issues Releases Activity

kern: unify all waiting semantics to use single api

Browse source
This commit is contained in:
Michael Scire 2021-09-19 10:11:56 -07:00 committed by SciresM
parent f6fb5f2c8d
commit 90732ff311
22 changed files with 904 additions and 683 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
libraries/libmesosphere/include/mesosphere/kern_k_light_condition_variable.hpp
View file

@ -27,59 +27,9 @@ namespace ams::kern {
KThread::WaiterList m_wait_list; KThread::WaiterList m_wait_list;
public: public:
constexpr ALWAYS_INLINE KLightConditionVariable() : m_wait_list() { /* ... */ } constexpr ALWAYS_INLINE KLightConditionVariable() : m_wait_list() { /* ... */ }
private:
void WaitImpl(KLightLock *lock, s64 timeout, bool allow_terminating_thread) {
KThread *owner = GetCurrentThreadPointer();
KHardwareTimer *timer;
/* Sleep the thread. */
{
KScopedSchedulerLockAndSleep lk(&timer, owner, timeout);
if (!allow_terminating_thread && owner->IsTerminationRequested()) {
lk.CancelSleep();
return;
}
lock->Unlock();
/* Set the thread as waiting. */
GetCurrentThread().SetState(KThread::ThreadState_Waiting);
/* Add the thread to the queue. */
m_wait_list.push_back(GetCurrentThread());
}
/* Remove the thread from the wait list. */
{
KScopedSchedulerLock sl;
m_wait_list.erase(m_wait_list.iterator_to(GetCurrentThread()));
}
/* Cancel the task that the sleep setup. */
if (timer != nullptr) {
timer->CancelTask(owner);
}
/* Re-acquire the lock. */
lock->Lock();
}
public: public:
void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true) { void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true);
this->WaitImpl(lock, timeout, allow_terminating_thread); void Broadcast();
}
void Broadcast() {
KScopedSchedulerLock lk;
/* Signal all threads. */
for (auto &thread : m_wait_list) {
thread.SetState(KThread::ThreadState_Runnable);
}
}
}; };
} }

6
libraries/libmesosphere/include/mesosphere/kern_k_light_server_session.hpp
View file

@ -27,12 +27,12 @@ namespace ams::kern {
MESOSPHERE_AUTOOBJECT_TRAITS(KLightServerSession, KAutoObject); MESOSPHERE_AUTOOBJECT_TRAITS(KLightServerSession, KAutoObject);
private: private:
KLightSession *m_parent; KLightSession *m_parent;
KThreadQueue m_request_queue; KThread::WaiterList m_request_list;
KThreadQueue m_server_queue;
KThread *m_current_request; KThread *m_current_request;
u64 m_server_thread_id;
KThread *m_server_thread; KThread *m_server_thread;
public: public:
constexpr KLightServerSession() : m_parent(), m_request_queue(), m_server_queue(), m_current_request(), m_server_thread() { /* ... */ } constexpr KLightServerSession() : m_parent(), m_request_list(), m_current_request(), m_server_thread_id(), m_server_thread() { /* ... */ }
void Initialize(KLightSession *parent) { void Initialize(KLightSession *parent) {
/* Set member variables. */ /* Set member variables. */

34
libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp
View file

@ -45,7 +45,39 @@ namespace ams::kern {
public: public:
virtual void Finalize() override; virtual void Finalize() override;
virtual bool IsSignaled() const { AMS_INFINITE_LOOP(); } virtual bool IsSignaled() const { AMS_INFINITE_LOOP(); }
virtual void DumpWaiters();
void DumpWaiters();
ALWAYS_INLINE void LinkNode(ThreadListNode *node) {
/* Link the node to the list. */
if (m_thread_list_tail == nullptr) {
m_thread_list_head = node;
} else {
m_thread_list_tail->next = node;
}
m_thread_list_tail = node;
}
ALWAYS_INLINE void UnlinkNode(ThreadListNode *node) {
/* Unlink the node from the list. */
ThreadListNode *prev_ptr = reinterpret_cast<ThreadListNode *>(std::addressof(m_thread_list_head));
ThreadListNode *prev_val = nullptr;
ThreadListNode *prev, *tail_prev;
do {
prev = prev_ptr;
prev_ptr = prev_ptr->next;
tail_prev = prev_val;
prev_val = prev_ptr;
} while (prev_ptr != node);
if (m_thread_list_tail == node) {
m_thread_list_tail = tail_prev;
}
prev->next = node->next;
}
}; };
} }

43
libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp
View file

@ -37,6 +37,7 @@ namespace ams::kern {
friend class KProcess; friend class KProcess;
friend class KConditionVariable; friend class KConditionVariable;
friend class KAddressArbiter; friend class KAddressArbiter;
friend class KThreadQueue;
public: public:
static constexpr s32 MainThreadPriority = 1; static constexpr s32 MainThreadPriority = 1;
static constexpr s32 IdleThreadPriority = 64; static constexpr s32 IdleThreadPriority = 64;
@ -191,7 +192,6 @@ namespace ams::kern {
KAffinityMask m_physical_affinity_mask{}; KAffinityMask m_physical_affinity_mask{};
u64 m_thread_id{}; u64 m_thread_id{};
std::atomic<s64> m_cpu_time{}; std::atomic<s64> m_cpu_time{};
KSynchronizationObject *m_synced_object{};
KProcessAddress m_address_key{}; KProcessAddress m_address_key{};
KProcess *m_parent{}; KProcess *m_parent{};
void *m_kernel_stack_top{}; void *m_kernel_stack_top{};
@ -204,9 +204,7 @@ namespace ams::kern {
s64 m_last_scheduled_tick{}; s64 m_last_scheduled_tick{};
QueueEntry m_per_core_priority_queue_entry[cpu::NumCores]{}; QueueEntry m_per_core_priority_queue_entry[cpu::NumCores]{};
KLightLock *m_waiting_lock{}; KLightLock *m_waiting_lock{};
KThreadQueue *m_wait_queue{};
KThreadQueue *m_sleeping_queue{};
WaiterList m_waiter_list{}; WaiterList m_waiter_list{};
WaiterList m_pinned_waiter_list{}; WaiterList m_pinned_waiter_list{};
KThread *m_lock_owner{}; KThread *m_lock_owner{};
@ -215,6 +213,7 @@ namespace ams::kern {
u32 m_address_key_value{}; u32 m_address_key_value{};
u32 m_suspend_request_flags{}; u32 m_suspend_request_flags{};
u32 m_suspend_allowed_flags{}; u32 m_suspend_allowed_flags{};
s32 m_synced_index{};
Result m_wait_result; Result m_wait_result;
Result m_debug_exception_result; Result m_debug_exception_result;
s32 m_base_priority{}; s32 m_base_priority{};
@ -374,6 +373,8 @@ namespace ams::kern {
void FinishTermination(); void FinishTermination();
void IncreaseBasePriority(s32 priority); void IncreaseBasePriority(s32 priority);
NOINLINE void SetState(ThreadState state);
public: public:
constexpr u64 GetThreadId() const { return m_thread_id; } constexpr u64 GetThreadId() const { return m_thread_id; }
@ -390,7 +391,6 @@ namespace ams::kern {
constexpr ThreadState GetState() const { return static_cast<ThreadState>(m_thread_state & ThreadState_Mask); } constexpr ThreadState GetState() const { return static_cast<ThreadState>(m_thread_state & ThreadState_Mask); }
constexpr ThreadState GetRawState() const { return m_thread_state; } constexpr ThreadState GetRawState() const { return m_thread_state; }
NOINLINE void SetState(ThreadState state);
NOINLINE KThreadContext *GetContextForSchedulerLoop(); NOINLINE KThreadContext *GetContextForSchedulerLoop();
@ -442,8 +442,6 @@ namespace ams::kern {
constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return m_per_core_priority_queue_entry[core]; } constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return m_per_core_priority_queue_entry[core]; }
constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return m_per_core_priority_queue_entry[core]; } constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return m_per_core_priority_queue_entry[core]; }
constexpr void SetSleepingQueue(KThreadQueue *q) { m_sleeping_queue = q; }
constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return m_condvar_tree; } constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return m_condvar_tree; }
constexpr s32 GetNumKernelWaiters() const { return m_num_kernel_waiters; } constexpr s32 GetNumKernelWaiters() const { return m_num_kernel_waiters; }
@ -460,29 +458,22 @@ namespace ams::kern {
constexpr void SetLockOwner(KThread *owner) { m_lock_owner = owner; } constexpr void SetLockOwner(KThread *owner) { m_lock_owner = owner; }
constexpr KThread *GetLockOwner() const { return m_lock_owner; } constexpr KThread *GetLockOwner() const { return m_lock_owner; }
constexpr void SetSyncedObject(KSynchronizationObject *obj, Result wait_res) { constexpr void ClearWaitQueue() { m_wait_queue = nullptr; }
MESOSPHERE_ASSERT_THIS();
m_synced_object = obj; void BeginWait(KThreadQueue *queue);
m_wait_result = wait_res; void NotifyAvailable(KSynchronizationObject *signaled_object, Result wait_result);
} void EndWait(Result wait_result);
void CancelWait(Result wait_result, bool cancel_timer_task);
constexpr Result GetWaitResult(KSynchronizationObject **out) const { constexpr void SetSyncedIndex(s32 index) { m_synced_index = index; }
MESOSPHERE_ASSERT_THIS(); constexpr s32 GetSyncedIndex() const { return m_synced_index; }
*out = m_synced_object; constexpr void SetWaitResult(Result wait_res) { m_wait_result = wait_res; }
return m_wait_result; constexpr Result GetWaitResult() const { return m_wait_result; }
}
constexpr void SetDebugExceptionResult(Result result) { constexpr void SetDebugExceptionResult(Result result) { m_debug_exception_result = result; }
MESOSPHERE_ASSERT_THIS();
m_debug_exception_result = result;
}
constexpr Result GetDebugExceptionResult() const { constexpr Result GetDebugExceptionResult() const { return m_debug_exception_result; }
MESOSPHERE_ASSERT_THIS();
return m_debug_exception_result;
}
void WaitCancel(); void WaitCancel();
@ -585,8 +576,6 @@ namespace ams::kern {
} }
} }
void Wakeup();
void SetBasePriority(s32 priority); void SetBasePriority(s32 priority);
Result SetPriorityToIdle(); Result SetPriorityToIdle();

65
libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp
View file

@ -16,69 +16,28 @@
#pragma once #pragma once
#include <mesosphere/kern_common.hpp> #include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_thread.hpp> #include <mesosphere/kern_k_thread.hpp>
#include <mesosphere/kern_select_hardware_timer.hpp>
namespace ams::kern { namespace ams::kern {
class KThreadQueue { class KThreadQueue {
private: private:
KThread::WaiterList m_wait_list; KHardwareTimer *m_hardware_timer;
public: public:
constexpr ALWAYS_INLINE KThreadQueue() : m_wait_list() { /* ... */ } constexpr ALWAYS_INLINE KThreadQueue() : m_hardware_timer(nullptr) { /* ... */ }
bool IsEmpty() const { return m_wait_list.empty(); } constexpr void SetHardwareTimer(KHardwareTimer *timer) { m_hardware_timer = timer; }
KThread::WaiterList::iterator begin() { return m_wait_list.begin(); } virtual void NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result);
KThread::WaiterList::iterator end() { return m_wait_list.end(); } virtual void EndWait(KThread *waiting_thread, Result wait_result);
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task);
};
bool SleepThread(KThread *t) { class KThreadQueueWithoutEndWait : public KThreadQueue {
KScopedSchedulerLock sl; public:
constexpr ALWAYS_INLINE KThreadQueueWithoutEndWait() : KThreadQueue() { /* ... */ }
/* If the thread needs terminating, don't enqueue it. */ virtual void EndWait(KThread *waiting_thread, Result wait_result) override final;
if (t->IsTerminationRequested()) {
return false;
}
/* Set the thread's queue and mark it as waiting. */
t->SetSleepingQueue(this);
t->SetState(KThread::ThreadState_Waiting);
/* Add the thread to the queue. */
m_wait_list.push_back(*t);
return true;
}
void WakeupThread(KThread *t) {
KScopedSchedulerLock sl;
/* Remove the thread from the queue. */
m_wait_list.erase(m_wait_list.iterator_to(*t));
/* Mark the thread as no longer sleeping. */
t->SetState(KThread::ThreadState_Runnable);
t->SetSleepingQueue(nullptr);
}
KThread *WakeupFrontThread() {
KScopedSchedulerLock sl;
if (m_wait_list.empty()) {
return nullptr;
} else {
/* Remove the thread from the queue. */
auto it = m_wait_list.begin();
KThread *thread = std::addressof(*it);
m_wait_list.erase(it);
MESOSPHERE_ASSERT(thread->GetState() == KThread::ThreadState_Waiting);
/* Mark the thread as no longer sleeping. */
thread->SetState(KThread::ThreadState_Runnable);
thread->SetSleepingQueue(nullptr);
return thread;
}
}
}; };
} }

7
libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp
View file

@ -20,14 +20,13 @@
namespace ams::kern { namespace ams::kern {
class KWaitObject : public KTimerTask { class KWaitObject {
private: private:
KThread::WaiterList m_wait_list; KThread::WaiterList m_wait_list;
bool m_timer_used; KThread *m_next_thread;
public: public:
constexpr KWaitObject() : m_wait_list(), m_timer_used() { /* ... */ } constexpr KWaitObject() : m_wait_list(), m_next_thread() { /* ... */ }
virtual void OnTimer() override;
Result Synchronize(s64 timeout); Result Synchronize(s64 timeout);
}; };

5
libraries/libmesosphere/include/mesosphere/kern_k_worker_task_manager.hpp
View file

@ -32,8 +32,7 @@ namespace ams::kern {
private: private:
KWorkerTask *m_head_task; KWorkerTask *m_head_task;
KWorkerTask *m_tail_task; KWorkerTask *m_tail_task;
KThread *m_thread; KThread *m_waiting_thread;
bool m_active;
private: private:
static void ThreadFunction(uintptr_t arg); static void ThreadFunction(uintptr_t arg);
void ThreadFunctionImpl(); void ThreadFunctionImpl();
@ -41,7 +40,7 @@ namespace ams::kern {
KWorkerTask *GetTask(); KWorkerTask *GetTask();
void AddTask(KWorkerTask *task); void AddTask(KWorkerTask *task);
public: public:
constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_thread(), m_active() { /* ... */ } constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_waiting_thread() { /* ... */ }
NOINLINE void Initialize(s32 priority); NOINLINE void Initialize(s32 priority);
static void AddTask(WorkerType type, KWorkerTask *task); static void AddTask(WorkerType type, KWorkerTask *task);

100
libraries/libmesosphere/source/kern_k_address_arbiter.cpp
View file

@ -43,6 +43,24 @@ namespace ams::kern {
return UserspaceAccess::UpdateIfEqualAtomic(out, GetPointer<s32>(address), value, new_value); return UserspaceAccess::UpdateIfEqualAtomic(out, GetPointer<s32>(address), value, new_value);
} }
class ThreadQueueImplForKAddressArbiter final : public KThreadQueue {
private:
KAddressArbiter::ThreadTree *m_tree;
public:
constexpr ThreadQueueImplForKAddressArbiter(KAddressArbiter::ThreadTree *t) : KThreadQueue(), m_tree(t) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* If the thread is waiting on an address arbiter, remove it from the tree. */
if (waiting_thread->IsWaitingForAddressArbiter()) {
m_tree->erase(m_tree->iterator_to(*waiting_thread));
waiting_thread->ClearAddressArbiter();
}
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
Result KAddressArbiter::Signal(uintptr_t addr, s32 count) { Result KAddressArbiter::Signal(uintptr_t addr, s32 count) {
@ -53,14 +71,14 @@ namespace ams::kern {
auto it = m_tree.nfind_key({ addr, -1 }); auto it = m_tree.nfind_key({ addr, -1 });
while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
/* End the thread's wait. */
KThread *target_thread = std::addressof(*it); KThread *target_thread = std::addressof(*it);
target_thread->SetSyncedObject(nullptr, ResultSuccess()); target_thread->EndWait(ResultSuccess());
AMS_ASSERT(target_thread->IsWaitingForAddressArbiter()); MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
target_thread->Wakeup(); target_thread->ClearAddressArbiter();
it = m_tree.erase(it); it = m_tree.erase(it);
target_thread->ClearAddressArbiter();
++num_waiters; ++num_waiters;
} }
} }
@ -80,14 +98,14 @@ namespace ams::kern {
auto it = m_tree.nfind_key({ addr, -1 }); auto it = m_tree.nfind_key({ addr, -1 });
while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
/* End the thread's wait. */
KThread *target_thread = std::addressof(*it); KThread *target_thread = std::addressof(*it);
target_thread->SetSyncedObject(nullptr, ResultSuccess()); target_thread->EndWait(ResultSuccess());
AMS_ASSERT(target_thread->IsWaitingForAddressArbiter()); MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
target_thread->Wakeup(); target_thread->ClearAddressArbiter();
it = m_tree.erase(it); it = m_tree.erase(it);
target_thread->ClearAddressArbiter();
++num_waiters; ++num_waiters;
} }
} }
@ -142,14 +160,14 @@ namespace ams::kern {
R_UNLESS(user_value == value, svc::ResultInvalidState()); R_UNLESS(user_value == value, svc::ResultInvalidState());
while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
/* End the thread's wait. */
KThread *target_thread = std::addressof(*it); KThread *target_thread = std::addressof(*it);
target_thread->SetSyncedObject(nullptr, ResultSuccess()); target_thread->EndWait(ResultSuccess());
AMS_ASSERT(target_thread->IsWaitingForAddressArbiter()); MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
target_thread->Wakeup(); target_thread->ClearAddressArbiter();
it = m_tree.erase(it); it = m_tree.erase(it);
target_thread->ClearAddressArbiter();
++num_waiters; ++num_waiters;
} }
} }
@ -160,6 +178,7 @@ namespace ams::kern {
/* Prepare to wait. */ /* Prepare to wait. */
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKAddressArbiter wait_queue(std::addressof(m_tree));
{ {
KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
@ -170,9 +189,6 @@ namespace ams::kern {
return svc::ResultTerminationRequested(); return svc::ResultTerminationRequested();
} }
/* Set the synced object. */
cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
/* Read the value from userspace. */ /* Read the value from userspace. */
s32 user_value; s32 user_value;
bool succeeded; bool succeeded;
@ -202,33 +218,21 @@ namespace ams::kern {
/* Set the arbiter. */ /* Set the arbiter. */
cur_thread->SetAddressArbiter(std::addressof(m_tree), addr); cur_thread->SetAddressArbiter(std::addressof(m_tree), addr);
m_tree.insert(*cur_thread); m_tree.insert(*cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
/* Wait for the thread to finish. */
wait_queue.SetHardwareTimer(timer);
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* Cancel the timer wait. */ /* Get the wait result. */
if (timer != nullptr) { return cur_thread->GetWaitResult();
timer->CancelTask(cur_thread);
}
/* Remove from the address arbiter. */
{
KScopedSchedulerLock sl;
if (cur_thread->IsWaitingForAddressArbiter()) {
m_tree.erase(m_tree.iterator_to(*cur_thread));
cur_thread->ClearAddressArbiter();
}
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
Result KAddressArbiter::WaitIfEqual(uintptr_t addr, s32 value, s64 timeout) { Result KAddressArbiter::WaitIfEqual(uintptr_t addr, s32 value, s64 timeout) {
/* Prepare to wait. */ /* Prepare to wait. */
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKAddressArbiter wait_queue(std::addressof(m_tree));
{ {
KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
@ -239,9 +243,6 @@ namespace ams::kern {
return svc::ResultTerminationRequested(); return svc::ResultTerminationRequested();
} }
/* Set the synced object. */
cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
/* Read the value from userspace. */ /* Read the value from userspace. */
s32 user_value; s32 user_value;
if (!ReadFromUser(std::addressof(user_value), addr)) { if (!ReadFromUser(std::addressof(user_value), addr)) {
@ -264,27 +265,14 @@ namespace ams::kern {
/* Set the arbiter. */ /* Set the arbiter. */
cur_thread->SetAddressArbiter(std::addressof(m_tree), addr); cur_thread->SetAddressArbiter(std::addressof(m_tree), addr);
m_tree.insert(*cur_thread); m_tree.insert(*cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
/* Wait for the thread to finish. */
wait_queue.SetHardwareTimer(timer);
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* Cancel the timer wait. */ /* Get the wait result. */
if (timer != nullptr) { return cur_thread->GetWaitResult();
timer->CancelTask(cur_thread);
}
/* Remove from the address arbiter. */
{
KScopedSchedulerLock sl;
if (cur_thread->IsWaitingForAddressArbiter()) {
m_tree.erase(m_tree.iterator_to(*cur_thread));
cur_thread->ClearAddressArbiter();
}
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
} }

20
libraries/libmesosphere/source/kern_k_client_session.cpp
View file

@ -40,17 +40,7 @@ namespace ams::kern {
request->Initialize(nullptr, address, size); request->Initialize(nullptr, address, size);
/* Send the request. */ /* Send the request. */
{ return m_parent->OnRequest(request);
KScopedSchedulerLock sl;
GetCurrentThread().SetSyncedObject(nullptr, ResultSuccess());
R_TRY(m_parent->OnRequest(request));
}
/* Get the result. */
KSynchronizationObject *dummy;
return GetCurrentThread().GetWaitResult(std::addressof(dummy));
} }
Result KClientSession::SendAsyncRequest(KEvent *event, uintptr_t address, size_t size) { Result KClientSession::SendAsyncRequest(KEvent *event, uintptr_t address, size_t size) {
@ -65,13 +55,7 @@ namespace ams::kern {
request->Initialize(event, address, size); request->Initialize(event, address, size);
/* Send the request. */ /* Send the request. */
{ return m_parent->OnRequest(request);
KScopedSchedulerLock sl;
R_TRY(m_parent->OnRequest(request));
}
return ResultSuccess();
} }
} }

143
libraries/libmesosphere/source/kern_k_condition_variable.cpp
View file

@ -31,10 +31,46 @@ namespace ams::kern {
return UserspaceAccess::UpdateLockAtomic(out, GetPointer<u32>(address), if_zero, new_orr_mask); return UserspaceAccess::UpdateLockAtomic(out, GetPointer<u32>(address), if_zero, new_orr_mask);
} }
class ThreadQueueImplForKConditionVariableWaitForAddress final : public KThreadQueue {
public:
constexpr ThreadQueueImplForKConditionVariableWaitForAddress() : KThreadQueue() { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread as a waiter from its owner. */
waiting_thread->GetLockOwner()->RemoveWaiter(waiting_thread);
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
class ThreadQueueImplForKConditionVariableWaitConditionVariable final : public KThreadQueue {
private:
KConditionVariable::ThreadTree *m_tree;
public:
constexpr ThreadQueueImplForKConditionVariableWaitConditionVariable(KConditionVariable::ThreadTree *t) : KThreadQueue(), m_tree(t) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread as a waiter from its owner. */
if (KThread *owner = waiting_thread->GetLockOwner(); owner != nullptr) {
owner->RemoveWaiter(waiting_thread);
}
/* If the thread is waiting on a condvar, remove it from the tree. */
if (waiting_thread->IsWaitingForConditionVariable()) {
m_tree->erase(m_tree->iterator_to(*waiting_thread));
waiting_thread->ClearConditionVariable();
}
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
Result KConditionVariable::SignalToAddress(KProcessAddress addr) { Result KConditionVariable::SignalToAddress(KProcessAddress addr) {
KThread *owner_thread = std::addressof(GetCurrentThread()); KThread *owner_thread = GetCurrentThreadPointer();
/* Signal the address. */ /* Signal the address. */
{ {
@ -46,44 +82,43 @@ namespace ams::kern {
/* Determine the next tag. */ /* Determine the next tag. */
u32 next_value = 0; u32 next_value = 0;
if (next_owner_thread) { if (next_owner_thread != nullptr) {
next_value = next_owner_thread->GetAddressKeyValue(); next_value = next_owner_thread->GetAddressKeyValue();
if (num_waiters > 1) { if (num_waiters > 1) {
next_value |= ams::svc::HandleWaitMask; next_value |= ams::svc::HandleWaitMask;
} }
next_owner_thread->SetSyncedObject(nullptr, ResultSuccess());
next_owner_thread->Wakeup();
}
/* Write the value to userspace. */ /* Write the value to userspace. */
if (!WriteToUser(addr, std::addressof(next_value))) { Result result;
if (next_owner_thread) { if (AMS_LIKELY(WriteToUser(addr, std::addressof(next_value)))) {
next_owner_thread->SetSyncedObject(nullptr, svc::ResultInvalidCurrentMemory()); result = ResultSuccess();
} else {
result = svc::ResultInvalidCurrentMemory();
} }
return svc::ResultInvalidCurrentMemory(); /* Signal the next owner thread. */
} next_owner_thread->EndWait(result);
} return result;
} else {
/* Just write the value to userspace. */
R_UNLESS(WriteToUser(addr, std::addressof(next_value)), svc::ResultInvalidCurrentMemory());
return ResultSuccess(); return ResultSuccess();
} }
}
}
Result KConditionVariable::WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value) { Result KConditionVariable::WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value) {
KThread *cur_thread = std::addressof(GetCurrentThread()); KThread *cur_thread = GetCurrentThreadPointer();
ThreadQueueImplForKConditionVariableWaitForAddress wait_queue;
/* Wait for the address. */ /* Wait for the address. */
{
KScopedAutoObject<KThread> owner_thread;
MESOSPHERE_ASSERT(owner_thread.IsNull());
{ {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
cur_thread->SetSyncedObject(nullptr, ResultSuccess());
/* Check if the thread should terminate. */ /* Check if the thread should terminate. */
R_UNLESS(!cur_thread->IsTerminationRequested(), svc::ResultTerminationRequested()); R_UNLESS(!cur_thread->IsTerminationRequested(), svc::ResultTerminationRequested());
{
/* Read the tag from userspace. */ /* Read the tag from userspace. */
u32 test_tag; u32 test_tag;
R_UNLESS(ReadFromUser(std::addressof(test_tag), addr), svc::ResultInvalidCurrentMemory()); R_UNLESS(ReadFromUser(std::addressof(test_tag), addr), svc::ResultInvalidCurrentMemory());
@ -92,29 +127,19 @@ namespace ams::kern {
R_SUCCEED_IF(test_tag != (handle | ams::svc::HandleWaitMask)); R_SUCCEED_IF(test_tag != (handle | ams::svc::HandleWaitMask));
/* Get the lock owner thread. */ /* Get the lock owner thread. */
owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(handle); KScopedAutoObject owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(handle);
R_UNLESS(owner_thread.IsNotNull(), svc::ResultInvalidHandle()); R_UNLESS(owner_thread.IsNotNull(), svc::ResultInvalidHandle());
/* Update the lock. */ /* Update the lock. */
cur_thread->SetAddressKey(addr, value); cur_thread->SetAddressKey(addr, value);
owner_thread->AddWaiter(cur_thread); owner_thread->AddWaiter(cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
}
}
MESOSPHERE_ASSERT(owner_thread.IsNotNull());
/* Remove the thread as a waiter from the lock owner. */ /* Begin waiting. */
{ cur_thread->BeginWait(std::addressof(wait_queue));
KScopedSchedulerLock sl;
if (KThread *mutex_owner = cur_thread->GetLockOwner(); mutex_owner != nullptr) {
mutex_owner->RemoveWaiter(cur_thread);
}
}
} }
/* Get the wait result. */ /* Get the wait result. */
KSynchronizationObject *dummy; return cur_thread->GetWaitResult();
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
void KConditionVariable::SignalImpl(KThread *thread) { void KConditionVariable::SignalImpl(KThread *thread) {
@ -139,8 +164,7 @@ namespace ams::kern {
if (AMS_LIKELY(can_access)) { if (AMS_LIKELY(can_access)) {
if (prev_tag == ams::svc::InvalidHandle) { if (prev_tag == ams::svc::InvalidHandle) {
/* If nobody held the lock previously, we're all good. */ /* If nobody held the lock previously, we're all good. */
thread->SetSyncedObject(nullptr, ResultSuccess()); thread->EndWait(ResultSuccess());
thread->Wakeup();
} else { } else {
/* Get the previous owner. */ /* Get the previous owner. */
KThread *owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(static_cast<ams::svc::Handle>(prev_tag & ~ams::svc::HandleWaitMask)) KThread *owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(static_cast<ams::svc::Handle>(prev_tag & ~ams::svc::HandleWaitMask))
@ -151,14 +175,12 @@ namespace ams::kern {
owner_thread->Close(); owner_thread->Close();
} else { } else {
/* The lock was tagged with a thread that doesn't exist. */ /* The lock was tagged with a thread that doesn't exist. */
thread->SetSyncedObject(nullptr, svc::ResultInvalidState()); thread->EndWait(svc::ResultInvalidState());
thread->Wakeup();
} }
} }
} else { } else {
/* If the address wasn't accessible, note so. */ /* If the address wasn't accessible, note so. */
thread->SetSyncedObject(nullptr, svc::ResultInvalidCurrentMemory()); thread->EndWait(svc::ResultInvalidCurrentMemory());
thread->Wakeup();
} }
} }
@ -190,13 +212,11 @@ namespace ams::kern {
/* Prepare to wait. */ /* Prepare to wait. */
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKConditionVariableWaitConditionVariable wait_queue(std::addressof(m_tree));
{ {
KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
/* Set the synced object. */
cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
/* Check that the thread isn't terminating. */ /* Check that the thread isn't terminating. */
if (cur_thread->IsTerminationRequested()) { if (cur_thread->IsTerminationRequested()) {
slp.CancelSleep(); slp.CancelSleep();
@ -219,8 +239,7 @@ namespace ams::kern {
} }
/* Wake up the next owner. */ /* Wake up the next owner. */
next_owner_thread->SetSyncedObject(nullptr, ResultSuccess()); next_owner_thread->EndWait(ResultSuccess());
next_owner_thread->Wakeup();
} }
/* Write to the cv key. */ /* Write to the cv key. */
@ -237,40 +256,20 @@ namespace ams::kern {
} }
} }
/* If timeout is zero, time out. */
R_UNLESS(timeout != 0, svc::ResultTimedOut());
/* Update condition variable tracking. */ /* Update condition variable tracking. */
{
cur_thread->SetConditionVariable(std::addressof(m_tree), addr, key, value); cur_thread->SetConditionVariable(std::addressof(m_tree), addr, key, value);
m_tree.insert(*cur_thread); m_tree.insert(*cur_thread);
/* Begin waiting. */
wait_queue.SetHardwareTimer(timer);
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* If the timeout is non-zero, set the thread as waiting. */ /* Get the wait result. */
if (timeout != 0) { return cur_thread->GetWaitResult();
cur_thread->SetState(KThread::ThreadState_Waiting);
}
}
/* Remove from the condition variable. */
{
KScopedSchedulerLock sl;
if (KThread *owner = cur_thread->GetLockOwner(); owner != nullptr) {
owner->RemoveWaiter(cur_thread);
}
if (cur_thread->IsWaitingForConditionVariable()) {
m_tree.erase(m_tree.iterator_to(*cur_thread));
cur_thread->ClearConditionVariable();
}
}
/* Cancel the timer wait. */
if (timer != nullptr) {
timer->CancelTask(cur_thread);
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
} }

12
libraries/libmesosphere/source/kern_k_light_client_session.cpp
View file

@ -37,17 +37,7 @@ namespace ams::kern {
cur_thread->SetLightSessionData(data); cur_thread->SetLightSessionData(data);
/* Send the request. */ /* Send the request. */
{ return m_parent->OnRequest(cur_thread);
KScopedSchedulerLock sl;
cur_thread->SetSyncedObject(nullptr, ResultSuccess());
R_TRY(m_parent->OnRequest(cur_thread));
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
} }

84
libraries/libmesosphere/source/kern_k_light_condition_variable.cpp Normal file
View file

@ -0,0 +1,84 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern {
namespace {
class ThreadQueueImplForKLightConditionVariable final : public KThreadQueue {
private:
KThread::WaiterList *m_wait_list;
bool m_allow_terminating_thread;
public:
constexpr ThreadQueueImplForKLightConditionVariable(KThread::WaiterList *wl, bool term) : KThreadQueue(), m_wait_list(wl), m_allow_terminating_thread(term) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Only process waits if we're allowed to. */
if (svc::ResultTerminationRequested::Includes(wait_result) && m_allow_terminating_thread) {
return;
}
/* Remove the thread from the waiting thread from the light condition variable. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KLightConditionVariable::Wait(KLightLock *lock, s64 timeout, bool allow_terminating_thread) {
/* Create thread queue. */
KThread *owner = GetCurrentThreadPointer();
KHardwareTimer *timer;
ThreadQueueImplForKLightConditionVariable wait_queue(std::addressof(m_wait_list), allow_terminating_thread);
/* Sleep the thread. */
{
KScopedSchedulerLockAndSleep lk(&timer, owner, timeout);
if (!allow_terminating_thread && owner->IsTerminationRequested()) {
lk.CancelSleep();
return;
}
lock->Unlock();
/* Add the thread to the queue. */
m_wait_list.push_back(*owner);
/* Begin waiting. */
wait_queue.SetHardwareTimer(timer);
owner->BeginWait(std::addressof(wait_queue));
}
/* Re-acquire the lock. */
lock->Lock();
}
void KLightConditionVariable::Broadcast() {
KScopedSchedulerLock lk;
/* Signal all threads. */
for (auto it = m_wait_list.begin(); it != m_wait_list.end(); it = m_wait_list.erase(it)) {
it->EndWait(ResultSuccess());
}
}
}

30
libraries/libmesosphere/source/kern_k_light_lock.cpp
View file

@ -17,8 +17,23 @@
namespace ams::kern { namespace ams::kern {
namespace {
class ThreadQueueImplForKLightLock final : public KThreadQueue {
public:
constexpr ThreadQueueImplForKLightLock() : KThreadQueue() { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Do nothing, waiting to acquire a light lock cannot be canceled. */
MESOSPHERE_UNUSED(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KLightLock::LockSlowPath(uintptr_t _owner, uintptr_t _cur_thread) { void KLightLock::LockSlowPath(uintptr_t _owner, uintptr_t _cur_thread) {
KThread *cur_thread = reinterpret_cast<KThread *>(_cur_thread); KThread *cur_thread = reinterpret_cast<KThread *>(_cur_thread);
ThreadQueueImplForKLightLock wait_queue;
/* Pend the current thread waiting on the owner thread. */ /* Pend the current thread waiting on the owner thread. */
{ {
@ -34,22 +49,13 @@ namespace ams::kern {
cur_thread->SetAddressKey(reinterpret_cast<uintptr_t>(std::addressof(m_tag))); cur_thread->SetAddressKey(reinterpret_cast<uintptr_t>(std::addressof(m_tag)));
owner_thread->AddWaiter(cur_thread); owner_thread->AddWaiter(cur_thread);
/* Set thread states. */ /* Begin waiting to hold the lock. */
cur_thread->SetState(KThread::ThreadState_Waiting); cur_thread->BeginWait(std::addressof(wait_queue));
if (owner_thread->IsSuspended()) { if (owner_thread->IsSuspended()) {
owner_thread->ContinueIfHasKernelWaiters(); owner_thread->ContinueIfHasKernelWaiters();
} }
} }
/* We're no longer waiting on the lock owner. */
{
KScopedSchedulerLock sl;
if (KThread *owner_thread = cur_thread->GetLockOwner(); AMS_UNLIKELY(owner_thread != nullptr)) {
owner_thread->RemoveWaiter(cur_thread);
}
}
} }
void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) { void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) {
@ -71,7 +77,7 @@ namespace ams::kern {
next_tag |= 0x1; next_tag |= 0x1;
} }
next_owner->SetState(KThread::ThreadState_Runnable); next_owner->EndWait(ResultSuccess());
if (next_owner->IsSuspended()) { if (next_owner->IsSuspended()) {
next_owner->ContinueIfHasKernelWaiters(); next_owner->ContinueIfHasKernelWaiters();

200
libraries/libmesosphere/source/kern_k_light_server_session.cpp
View file

@ -17,6 +17,64 @@
namespace ams::kern { namespace ams::kern {
namespace {
constexpr u64 InvalidThreadId = -1ull;
class ThreadQueueImplForKLightServerSessionRequest final : public KThreadQueue {
private:
KThread::WaiterList *m_wait_list;
public:
constexpr ThreadQueueImplForKLightServerSessionRequest(KThread::WaiterList *wl) : KThreadQueue(), m_wait_list(wl) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Remove the thread from our wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread from our wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
class ThreadQueueImplForKLightServerSessionReceive final : public KThreadQueue {
private:
KThread **m_server_thread;
public:
constexpr ThreadQueueImplForKLightServerSessionReceive(KThread **st) : KThreadQueue(), m_server_thread(st) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Clear the server thread. */
*m_server_thread = nullptr;
/* Set the waiting thread as not cancelable. */
waiting_thread->ClearCancellable();
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Clear the server thread. */
*m_server_thread = nullptr;
/* Set the waiting thread as not cancelable. */
waiting_thread->ClearCancellable();
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KLightServerSession::Destroy() { void KLightServerSession::Destroy() {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
@ -33,31 +91,45 @@ namespace ams::kern {
Result KLightServerSession::OnRequest(KThread *request_thread) { Result KLightServerSession::OnRequest(KThread *request_thread) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
ThreadQueueImplForKLightServerSessionRequest wait_queue(std::addressof(m_request_list));
/* Send the request. */
{
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Check that the server isn't closed. */ /* Check that the server isn't closed. */
R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed());
/* Try to sleep the thread. */ /* Check that the request thread isn't terminating. */
R_UNLESS(m_request_queue.SleepThread(request_thread), svc::ResultTerminationRequested()); R_UNLESS(!request_thread->IsTerminationRequested(), svc::ResultTerminationRequested());
/* If we don't have a current request, wake up a server thread to handle it. */ /* Add the request thread to our list. */
if (m_current_request == nullptr) { m_request_list.push_back(*request_thread);
m_server_queue.WakeupFrontThread();
/* Begin waiting on the request. */
request_thread->BeginWait(std::addressof(wait_queue));
/* If we have a server thread, end its wait. */
if (m_server_thread != nullptr) {
m_server_thread->EndWait(ResultSuccess());
}
} }
return ResultSuccess(); /* NOTE: Nintendo returns GetCurrentThread().GetWaitResult() here. */
/* This is technically incorrect, although it doesn't cause problems in practice */
/* because this is only ever called with request_thread = GetCurrentThreadPointer(). */
return request_thread->GetWaitResult();
} }
Result KLightServerSession::ReplyAndReceive(u32 *data) { Result KLightServerSession::ReplyAndReceive(u32 *data) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
/* Set the server context. */ /* Set the server context. */
KThread *server_thread = GetCurrentThreadPointer(); GetCurrentThread().SetLightSessionData(data);
server_thread->SetLightSessionData(data);
/* Reply, if we need to. */ /* Reply, if we need to. */
KThread *cur_request = nullptr;
if (data[0] & KLightSession::ReplyFlag) { if (data[0] & KLightSession::ReplyFlag) {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
@ -68,78 +140,85 @@ namespace ams::kern {
/* Check that we have a request to reply to. */ /* Check that we have a request to reply to. */
R_UNLESS(m_current_request != nullptr, svc::ResultInvalidState()); R_UNLESS(m_current_request != nullptr, svc::ResultInvalidState());
/* Check that the server thread is correct. */ /* Check that the server thread id is correct. */
R_UNLESS(m_server_thread == server_thread, svc::ResultInvalidState()); R_UNLESS(m_server_thread_id == GetCurrentThread().GetId(), svc::ResultInvalidState());
/* If we can reply, do so. */ /* If we can reply, do so. */
if (!m_current_request->IsTerminationRequested()) { if (!m_current_request->IsTerminationRequested()) {
MESOSPHERE_ASSERT(m_current_request->GetState() == KThread::ThreadState_Waiting); std::memcpy(m_current_request->GetLightSessionData(), GetCurrentThread().GetLightSessionData(), KLightSession::DataSize);
MESOSPHERE_ASSERT(m_request_queue.begin() != m_request_queue.end() && m_current_request == std::addressof(*m_request_queue.begin())); m_current_request->EndWait(ResultSuccess());
std::memcpy(m_current_request->GetLightSessionData(), server_thread->GetLightSessionData(), KLightSession::DataSize);
m_request_queue.WakeupThread(m_current_request);
} }
/* Close our current request. */
m_current_request->Close();
/* Clear our current request. */ /* Clear our current request. */
cur_request = m_current_request;
m_current_request = nullptr; m_current_request = nullptr;
m_server_thread = nullptr; m_server_thread_id = InvalidThreadId;
} }
/* Close the current request, if we had one. */ /* Close any pending objects before we wait. */
if (cur_request != nullptr) { GetCurrentThread().DestroyClosedObjects();
cur_request->Close();
} /* Create the wait queue for our receive. */
ThreadQueueImplForKLightServerSessionReceive wait_queue(std::addressof(m_server_thread));
/* Receive. */ /* Receive. */
bool set_cancellable = false;
while (true) { while (true) {
/* Try to receive a request. */
{
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
/* Check that we aren't already receiving. */ /* Check that we aren't already receiving. */
R_UNLESS(m_server_queue.IsEmpty(), svc::ResultInvalidState());
R_UNLESS(m_server_thread == nullptr, svc::ResultInvalidState()); R_UNLESS(m_server_thread == nullptr, svc::ResultInvalidState());
R_UNLESS(m_server_thread_id == InvalidThreadId, svc::ResultInvalidState());
/* If we cancelled in a previous loop, clear cancel state. */
if (set_cancellable) {
server_thread->ClearCancellable();
set_cancellable = false;
}
/* Check that we're open. */ /* Check that we're open. */
R_UNLESS(!m_parent->IsClientClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsClientClosed(), svc::ResultSessionClosed());
R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed());
/* Check that we're not terminating. */
R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
/* If we have a request available, use it. */ /* If we have a request available, use it. */
if (m_current_request == nullptr && !m_request_queue.IsEmpty()) { if (auto head = m_request_list.begin(); head != m_request_list.end()) {
m_current_request = std::addressof(*m_request_queue.begin()); /* Set our current request. */
m_current_request = std::addressof(*head);
m_current_request->Open(); m_current_request->Open();
m_server_thread = server_thread;
break; /* Set our server thread id. */
} else { m_server_thread_id = GetCurrentThread().GetId();
/* Otherwise, wait for a request to come in. */
R_UNLESS(m_server_queue.SleepThread(server_thread), svc::ResultTerminationRequested()); /* Copy the client request data. */
std::memcpy(GetCurrentThread().GetLightSessionData(), m_current_request->GetLightSessionData(), KLightSession::DataSize);
/* We successfully received. */
return ResultSuccess();
}
/* We need to wait for a request to come in. */
/* Check if we were cancelled. */ /* Check if we were cancelled. */
if (server_thread->IsWaitCancelled()) { if (GetCurrentThread().IsWaitCancelled()) {
m_server_queue.WakeupThread(server_thread); GetCurrentThread().ClearWaitCancelled();
server_thread->ClearWaitCancelled();
return svc::ResultCancelled(); return svc::ResultCancelled();
} }
/* Otherwise, mark as cancellable. */ /* Mark ourselves as cancellable. */
server_thread->SetCancellable(); GetCurrentThread().SetCancellable();
set_cancellable = true;
} /* Wait for a request to come in. */
m_server_thread = GetCurrentThreadPointer();
GetCurrentThread().BeginWait(std::addressof(wait_queue));
} }
/* Copy the client data. */ /* We waited to receive a request; if our wait failed, return the failing result. */
std::memcpy(server_thread->GetLightSessionData(), m_current_request->GetLightSessionData(), KLightSession::DataSize); R_TRY(GetCurrentThread().GetWaitResult());
return ResultSuccess(); }
} }
void KLightServerSession::CleanupRequests() { void KLightServerSession::CleanupRequests() {
/* Cleanup all pending requests. */ /* Cleanup all pending requests. */
KThread *cur_request = nullptr;
{ {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
@ -147,34 +226,25 @@ namespace ams::kern {
if (m_current_request != nullptr) { if (m_current_request != nullptr) {
/* Reply to the current request. */ /* Reply to the current request. */
if (!m_current_request->IsTerminationRequested()) { if (!m_current_request->IsTerminationRequested()) {
MESOSPHERE_ASSERT(m_current_request->GetState() == KThread::ThreadState_Waiting); m_current_request->EndWait(svc::ResultSessionClosed());
MESOSPHERE_ASSERT(m_request_queue.begin() != m_request_queue.end() && m_current_request == std::addressof(*m_request_queue.begin()));
m_request_queue.WakeupThread(m_current_request);
m_current_request->SetSyncedObject(nullptr, svc::ResultSessionClosed());
} }
/* Clear our current request. */ /* Clear our current request. */
cur_request = m_current_request; m_current_request->Close();
m_current_request = nullptr; m_current_request = nullptr;
m_server_thread = nullptr; m_server_thread_id = InvalidThreadId;
} }
/* Reply to all other requests. */ /* Reply to all other requests. */
while (!m_request_queue.IsEmpty()) { for (auto &thread : m_request_list) {
KThread *client_thread = m_request_queue.WakeupFrontThread(); thread.EndWait(svc::ResultSessionClosed());
client_thread->SetSyncedObject(nullptr, svc::ResultSessionClosed());
} }
/* Wake up all server threads. */ /* Wait up our server thread, if we have one. */
while (!m_server_queue.IsEmpty()) { if (m_server_thread != nullptr) {
m_server_queue.WakeupFrontThread(); m_server_thread->EndWait(svc::ResultSessionClosed());
} }
} }
/* Close the current request, if we had one. */
if (cur_request != nullptr) {
cur_request->Close();
}
} }
} }

67
libraries/libmesosphere/source/kern_k_process.cpp
View file

@ -87,6 +87,29 @@ namespace ams::kern {
return ResultSuccess(); return ResultSuccess();
} }
class ThreadQueueImplForKProcessEnterUserException final : public KThreadQueue {
private:
KThread **m_exception_thread;
public:
constexpr ThreadQueueImplForKProcessEnterUserException(KThread **t) : KThreadQueue(), m_exception_thread(t) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Set the exception thread. */
*m_exception_thread = waiting_thread;
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread as a waiter on its mutex owner. */
waiting_thread->GetLockOwner()->RemoveWaiter(waiting_thread);
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
void KProcess::Finalize() { void KProcess::Finalize() {
@ -784,43 +807,43 @@ namespace ams::kern {
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
MESOSPHERE_ASSERT(this == cur_thread->GetOwnerProcess()); MESOSPHERE_ASSERT(this == cur_thread->GetOwnerProcess());
/* Try to claim the exception thread. */ /* Check that we haven't already claimed the exception thread. */
if (m_exception_thread != cur_thread) { if (m_exception_thread == cur_thread) {
const uintptr_t address_key = reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)); return false;
while (true) { }
/* Create the wait queue we'll be using. */
ThreadQueueImplForKProcessEnterUserException wait_queue(std::addressof(m_exception_thread));
/* Claim the exception thread. */
{ {
/* Lock the scheduler. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
/* If the thread is terminating, it can't enter. */ /* Check that we're not terminating. */
if (cur_thread->IsTerminationRequested()) { if (cur_thread->IsTerminationRequested()) {
return false; return false;
} }
/* If we have no exception thread, we succeeded. */ /* If we don't have an exception thread, we can just claim it directly. */
if (m_exception_thread == nullptr) { if (m_exception_thread == nullptr) {
m_exception_thread = cur_thread; m_exception_thread = cur_thread;
KScheduler::SetSchedulerUpdateNeeded(); KScheduler::SetSchedulerUpdateNeeded();
return true; return true;
} }
/* Otherwise, wait for us to not have an exception thread. */ /* Otherwise, we need to wait until we don't have an exception thread. */
cur_thread->SetAddressKey(address_key | 1);
/* Add the current thread as a waiter on the current exception thread. */
cur_thread->SetAddressKey(reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)) | 1);
m_exception_thread->AddWaiter(cur_thread); m_exception_thread->AddWaiter(cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
/* Wait to claim the exception thread. */
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* Remove the thread as a waiter from the lock owner. */ /* If our wait didn't end due to thread termination, we succeeded. */
{ return !svc::ResultTerminationRequested::Includes(cur_thread->GetWaitResult());
KScopedSchedulerLock sl;
if (KThread *owner_thread = cur_thread->GetLockOwner(); owner_thread != nullptr) {
owner_thread->RemoveWaiter(cur_thread);
}
}
}
} else {
return false;
}
} }
bool KProcess::LeaveUserException() { bool KProcess::LeaveUserException() {
@ -836,7 +859,7 @@ namespace ams::kern {
/* Remove waiter thread. */ /* Remove waiter thread. */
s32 num_waiters; s32 num_waiters;
if (KThread *next = thread->RemoveWaiterByKey(std::addressof(num_waiters), reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)) | 1); next != nullptr) { if (KThread *next = thread->RemoveWaiterByKey(std::addressof(num_waiters), reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)) | 1); next != nullptr) {
next->SetState(KThread::ThreadState_Runnable); next->EndWait(ResultSuccess());
} }
KScheduler::SetSchedulerUpdateNeeded(); KScheduler::SetSchedulerUpdateNeeded();

53
libraries/libmesosphere/source/kern_k_server_session.cpp
View file

@ -30,6 +30,8 @@ namespace ams::kern {
constexpr inline size_t PointerTransferBufferAlignment = 0x10; constexpr inline size_t PointerTransferBufferAlignment = 0x10;
class ThreadQueueImplForKServerSessionRequest final : public KThreadQueue { /* ... */ };
class ReceiveList { class ReceiveList {
private: private:
u32 m_data[ipc::MessageBuffer::MessageHeader::ReceiveListCountType_CountMax * ipc::MessageBuffer::ReceiveListEntry::GetDataSize() / sizeof(u32)]; u32 m_data[ipc::MessageBuffer::MessageHeader::ReceiveListCountType_CountMax * ipc::MessageBuffer::ReceiveListEntry::GetDataSize() / sizeof(u32)];
@ -1042,11 +1044,11 @@ namespace ams::kern {
/* Signal the event. */ /* Signal the event. */
event->Signal(); event->Signal();
} else { } else {
/* Set the thread as runnable. */ /* End the client thread's wait. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
if (client_thread->GetState() == KThread::ThreadState_Waiting) {
client_thread->SetSyncedObject(nullptr, result_for_client); if (!client_thread->IsTerminationRequested()) {
client_thread->SetState(KThread::ThreadState_Runnable); client_thread->EndWait(result_for_client);
} }
} }
} }
@ -1146,11 +1148,11 @@ namespace ams::kern {
/* Signal the event. */ /* Signal the event. */
event->Signal(); event->Signal();
} else { } else {
/* Set the thread as runnable. */ /* End the client thread's wait. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
if (client_thread->GetState() == KThread::ThreadState_Waiting) {
client_thread->SetSyncedObject(nullptr, client_result); if (!client_thread->IsTerminationRequested()) {
client_thread->SetState(KThread::ThreadState_Runnable); client_thread->EndWait(client_result);
} }
} }
} }
@ -1160,17 +1162,20 @@ namespace ams::kern {
Result KServerSession::OnRequest(KSessionRequest *request) { Result KServerSession::OnRequest(KSessionRequest *request) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
/* Create the wait queue. */
ThreadQueueImplForKServerSessionRequest wait_queue;
/* Handle the request. */
{
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Ensure that we can handle new requests. */ /* Ensure that we can handle new requests. */
R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed());
/* If there's no event, this is synchronous, so we should check for thread termination. */ /* Check that we're not terminating. */
if (request->GetEvent() == nullptr) { R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
KThread *thread = request->GetThread();
R_UNLESS(!thread->IsTerminationRequested(), svc::ResultTerminationRequested());
thread->SetState(KThread::ThreadState_Waiting);
}
/* Get whether we're empty. */ /* Get whether we're empty. */
const bool was_empty = m_request_list.empty(); const bool was_empty = m_request_list.empty();
@ -1184,7 +1189,14 @@ namespace ams::kern {
this->NotifyAvailable(); this->NotifyAvailable();
} }
return ResultSuccess(); /* If we have a request, this is asynchronous, and we don't need to wait. */
R_SUCCEED_IF(request->GetEvent() != nullptr);
/* This is a synchronous request, so we should wait for our request to complete. */
GetCurrentThread().BeginWait(std::addressof(wait_queue));
}
return GetCurrentThread().GetWaitResult();
} }
bool KServerSession::IsSignaledImpl() const { bool KServerSession::IsSignaledImpl() const {
@ -1264,11 +1276,11 @@ namespace ams::kern {
/* Signal the event. */ /* Signal the event. */
event->Signal(); event->Signal();
} else { } else {
/* Set the thread as runnable. */ /* End the client thread's wait. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
if (client_thread->GetState() == KThread::ThreadState_Waiting) {
client_thread->SetSyncedObject(nullptr, (R_SUCCEEDED(result) ? svc::ResultSessionClosed() : result)); if (!client_thread->IsTerminationRequested()) {
client_thread->SetState(KThread::ThreadState_Runnable); client_thread->EndWait(R_SUCCEEDED(result) ? svc::ResultSessionClosed() : result);
} }
} }
} }
@ -1310,6 +1322,7 @@ namespace ams::kern {
request->ClearEvent(); request->ClearEvent();
terminate = true; terminate = true;
} }
prev_request = request; prev_request = request;
} else if (!m_request_list.empty()) { } else if (!m_request_list.empty()) {
/* Pop the request from the front of the list. */ /* Pop the request from the front of the list. */

138
libraries/libmesosphere/source/kern_k_synchronization_object.cpp
View file

@ -17,6 +17,57 @@
namespace ams::kern { namespace ams::kern {
namespace {
class ThreadQueueImplForKSynchronizationObjectWait final : public KThreadQueueWithoutEndWait {
private:
using ThreadListNode = KSynchronizationObject::ThreadListNode;
private:
KSynchronizationObject **m_objects;
ThreadListNode *m_nodes;
s32 m_count;
public:
constexpr ThreadQueueImplForKSynchronizationObjectWait(KSynchronizationObject **o, ThreadListNode *n, s32 c) : m_objects(o), m_nodes(n), m_count(c) { /* ... */ }
virtual void NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result) override {
/* Determine the sync index, and unlink all nodes. */
s32 sync_index = -1;
for (auto i = 0; i < m_count; ++i) {
/* Check if this is the signaled object. */
if (m_objects[i] == signaled_object && sync_index == -1) {
sync_index = i;
}
/* Unlink the current node from the current object. */
m_objects[i]->UnlinkNode(std::addressof(m_nodes[i]));
}
/* Set the waiting thread's sync index. */
waiting_thread->SetSyncedIndex(sync_index);
/* Set the waiting thread as not cancellable. */
waiting_thread->ClearCancellable();
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove all nodes from our list. */
for (auto i = 0; i < m_count; ++i) {
m_objects[i]->UnlinkNode(std::addressof(m_nodes[i]));
}
/* Set the waiting thread as not cancellable. */
waiting_thread->ClearCancellable();
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KSynchronizationObject::Finalize() { void KSynchronizationObject::Finalize() {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
@ -43,14 +94,21 @@ namespace ams::kern {
/* Prepare for wait. */ /* Prepare for wait. */
KThread *thread = GetCurrentThreadPointer(); KThread *thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKSynchronizationObjectWait wait_queue(objects, thread_nodes, num_objects);
{ {
/* Setup the scheduling lock and sleep. */ /* Setup the scheduling lock and sleep. */
KScopedSchedulerLockAndSleep slp(std::addressof(timer), thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), thread, timeout);
/* Check if the thread should terminate. */
if (thread->IsTerminationRequested()) {
slp.CancelSleep();
return svc::ResultTerminationRequested();
}
/* Check if any of the objects are already signaled. */ /* Check if any of the objects are already signaled. */
for (auto i = 0; i < num_objects; ++i) { for (auto i = 0; i < num_objects; ++i) {
AMS_ASSERT(objects[i] != nullptr); MESOSPHERE_ASSERT(objects[i] != nullptr);
if (objects[i]->IsSignaled()) { if (objects[i]->IsSignaled()) {
*out_index = i; *out_index = i;
@ -65,12 +123,6 @@ namespace ams::kern {
return svc::ResultTimedOut(); return svc::ResultTimedOut();
} }
/* Check if the thread should terminate. */
if (thread->IsTerminationRequested()) {
slp.CancelSleep();
return svc::ResultTerminationRequested();
}
/* Check if waiting was canceled. */ /* Check if waiting was canceled. */
if (thread->IsWaitCancelled()) { if (thread->IsWaitCancelled()) {
slp.CancelSleep(); slp.CancelSleep();
@ -83,67 +135,25 @@ namespace ams::kern {
thread_nodes[i].thread = thread; thread_nodes[i].thread = thread;
thread_nodes[i].next = nullptr; thread_nodes[i].next = nullptr;
if (objects[i]->m_thread_list_tail == nullptr) { objects[i]->LinkNode(std::addressof(thread_nodes[i]));
objects[i]->m_thread_list_head = std::addressof(thread_nodes[i]);
} else {
objects[i]->m_thread_list_tail->next = std::addressof(thread_nodes[i]);
} }
objects[i]->m_thread_list_tail = std::addressof(thread_nodes[i]); /* Mark the thread as cancellable. */
}
/* Mark the thread as waiting. */
thread->SetCancellable(); thread->SetCancellable();
thread->SetSyncedObject(nullptr, svc::ResultTimedOut());
thread->SetState(KThread::ThreadState_Waiting); /* Clear the thread's synced index. */
thread->SetSyncedIndex(-1);
/* Wait for an object to be signaled. */
wait_queue.SetHardwareTimer(timer);
thread->BeginWait(std::addressof(wait_queue));
} }
/* The lock/sleep is done, so we should be able to get our result. */ /* Set the output index. */
*out_index = thread->GetSyncedIndex();
/* Thread is no longer cancellable. */
thread->ClearCancellable();
/* Cancel the timer as needed. */
if (timer != nullptr) {
timer->CancelTask(thread);
}
/* Get the wait result. */ /* Get the wait result. */
Result wait_result; return thread->GetWaitResult();
s32 sync_index = -1;
{
KScopedSchedulerLock lk;
KSynchronizationObject *synced_obj;
wait_result = thread->GetWaitResult(std::addressof(synced_obj));
for (auto i = 0; i < num_objects; ++i) {
/* Unlink the object from the list. */
ThreadListNode *prev_ptr = reinterpret_cast<ThreadListNode *>(std::addressof(objects[i]->m_thread_list_head));
ThreadListNode *prev_val = nullptr;
ThreadListNode *prev, *tail_prev;
do {
prev = prev_ptr;
prev_ptr = prev_ptr->next;
tail_prev = prev_val;
prev_val = prev_ptr;
} while (prev_ptr != std::addressof(thread_nodes[i]));
if (objects[i]->m_thread_list_tail == std::addressof(thread_nodes[i])) {
objects[i]->m_thread_list_tail = tail_prev;
}
prev->next = thread_nodes[i].next;
if (objects[i] == synced_obj) {
sync_index = i;
}
}
}
/* Set output. */
*out_index = sync_index;
return wait_result;
} }
void KSynchronizationObject::NotifyAvailable(Result result) { void KSynchronizationObject::NotifyAvailable(Result result) {
@ -158,11 +168,7 @@ namespace ams::kern {
/* Iterate over each thread. */ /* Iterate over each thread. */
for (auto *cur_node = m_thread_list_head; cur_node != nullptr; cur_node = cur_node->next) { for (auto *cur_node = m_thread_list_head; cur_node != nullptr; cur_node = cur_node->next) {
KThread *thread = cur_node->thread; cur_node->thread->NotifyAvailable(this, result);
if (thread->GetState() == KThread::ThreadState_Waiting) {
thread->SetSyncedObject(this, result);
thread->SetState(KThread::ThreadState_Runnable);
}
} }
} }

174
libraries/libmesosphere/source/kern_k_thread.cpp
View file

@ -47,6 +47,23 @@ namespace ams::kern {
KPageBuffer::Free(KPageBuffer::FromPhysicalAddress(stack_paddr)); KPageBuffer::Free(KPageBuffer::FromPhysicalAddress(stack_paddr));
} }
class ThreadQueueImplForKThreadSleep final : public KThreadQueueWithoutEndWait { /* ... */ };
class ThreadQueueImplForKThreadSetProperty final : public KThreadQueue {
private:
KThread::WaiterList *m_wait_list;
public:
constexpr ThreadQueueImplForKThreadSetProperty(KThread::WaiterList *wl) : m_wait_list(wl) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread from the wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type) { Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type) {
@ -131,12 +148,12 @@ namespace ams::kern {
m_priority = prio; m_priority = prio;
m_base_priority = prio; m_base_priority = prio;
/* Set sync object and waiting lock to null. */ /* Set waiting lock to null. */
m_synced_object = nullptr;
m_waiting_lock = nullptr; m_waiting_lock = nullptr;
/* Initialize sleeping queue. */ /* Initialize wait queue/sync index. */
m_sleeping_queue = nullptr; m_synced_index = -1;
m_wait_queue = nullptr;
/* Set suspend flags. */ /* Set suspend flags. */
m_suspend_request_flags = 0; m_suspend_request_flags = 0;
@ -295,12 +312,20 @@ namespace ams::kern {
auto it = m_waiter_list.begin(); auto it = m_waiter_list.begin();
while (it != m_waiter_list.end()) { while (it != m_waiter_list.end()) {
/* Get the thread. */
KThread * const waiter = std::addressof(*it);
/* The thread shouldn't be a kernel waiter. */ /* The thread shouldn't be a kernel waiter. */
MESOSPHERE_ASSERT(!IsKernelAddressKey(it->GetAddressKey())); MESOSPHERE_ASSERT(!IsKernelAddressKey(waiter->GetAddressKey()));
it->SetLockOwner(nullptr);
it->SetSyncedObject(nullptr, svc::ResultInvalidState()); /* Clear the lock owner. */
it->Wakeup(); waiter->SetLockOwner(nullptr);
/* Erase the waiter from our list. */
it = m_waiter_list.erase(it); it = m_waiter_list.erase(it);
/* Cancel the thread's wait. */
waiter->CancelWait(svc::ResultInvalidState(), true);
} }
} }
@ -320,24 +345,14 @@ namespace ams::kern {
return m_signaled; return m_signaled;
} }
void KThread::Wakeup() {
MESOSPHERE_ASSERT_THIS();
KScopedSchedulerLock sl;
if (this->GetState() == ThreadState_Waiting) {
if (m_sleeping_queue != nullptr) {
m_sleeping_queue->WakeupThread(this);
} else {
this->SetState(ThreadState_Runnable);
}
}
}
void KThread::OnTimer() { void KThread::OnTimer() {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
this->Wakeup(); /* If we're waiting, cancel the wait. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->CancelWait(this, svc::ResultTimedOut(), false);
}
} }
void KThread::StartTermination() { void KThread::StartTermination() {
@ -362,7 +377,7 @@ namespace ams::kern {
/* Signal. */ /* Signal. */
m_signaled = true; m_signaled = true;
this->NotifyAvailable(); KSynchronizationObject::NotifyAvailable();
/* Call the on thread termination handler. */ /* Call the on thread termination handler. */
KThreadContext::OnThreadTerminating(this); KThreadContext::OnThreadTerminating(this);
@ -496,10 +511,8 @@ namespace ams::kern {
} }
/* Resume any threads that began waiting on us while we were pinned. */ /* Resume any threads that began waiting on us while we were pinned. */
for (auto it = m_pinned_waiter_list.begin(); it != m_pinned_waiter_list.end(); ++it) { for (auto it = m_pinned_waiter_list.begin(); it != m_pinned_waiter_list.end(); it = m_pinned_waiter_list.erase(it)) {
if (it->GetState() == ThreadState_Waiting) { it->EndWait(ResultSuccess());
it->SetState(ThreadState_Runnable);
}
} }
} }
@ -646,9 +659,9 @@ namespace ams::kern {
} }
/* Update the pinned waiter list. */ /* Update the pinned waiter list. */
ThreadQueueImplForKThreadSetProperty wait_queue(std::addressof(m_pinned_waiter_list));
{ {
bool retry_update; bool retry_update;
bool thread_is_pinned = false;
do { do {
/* Lock the scheduler. */ /* Lock the scheduler. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
@ -676,27 +689,15 @@ namespace ams::kern {
/* Verify that the current thread isn't terminating. */ /* Verify that the current thread isn't terminating. */
R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested()); R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
/* Note that the thread was pinned. */
thread_is_pinned = true;
/* Wait until the thread isn't pinned any more. */ /* Wait until the thread isn't pinned any more. */
m_pinned_waiter_list.push_back(GetCurrentThread()); m_pinned_waiter_list.push_back(GetCurrentThread());
GetCurrentThread().SetState(ThreadState_Waiting); GetCurrentThread().BeginWait(std::addressof(wait_queue));
} else { } else {
/* If the thread isn't pinned, release the scheduler lock and retry until it's not current. */ /* If the thread isn't pinned, release the scheduler lock and retry until it's not current. */
retry_update = true; retry_update = true;
} }
} }
} while (retry_update); } while (retry_update);
/* If the thread was pinned, it no longer is, and we should remove the current thread from our waiter list. */
if (thread_is_pinned) {
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Remove from the list. */
m_pinned_waiter_list.erase(m_pinned_waiter_list.iterator_to(GetCurrentThread()));
}
} }
return ResultSuccess(); return ResultSuccess();
@ -785,14 +786,8 @@ namespace ams::kern {
/* Check if we're waiting and cancellable. */ /* Check if we're waiting and cancellable. */
if (this->GetState() == ThreadState_Waiting && m_cancellable) { if (this->GetState() == ThreadState_Waiting && m_cancellable) {
if (m_sleeping_queue != nullptr) {
m_sleeping_queue->WakeupThread(this);
m_wait_cancelled = true;
} else {
this->SetSyncedObject(nullptr, svc::ResultCancelled());
this->SetState(ThreadState_Runnable);
m_wait_cancelled = false; m_wait_cancelled = false;
} m_wait_queue->CancelWait(this, svc::ResultCancelled(), true);
} else { } else {
/* Otherwise, note that we cancelled a wait. */ /* Otherwise, note that we cancelled a wait. */
m_wait_cancelled = true; m_wait_cancelled = true;
@ -894,7 +889,8 @@ namespace ams::kern {
/* If the thread is now paused, update the pinned waiter list. */ /* If the thread is now paused, update the pinned waiter list. */
if (activity == ams::svc::ThreadActivity_Paused) { if (activity == ams::svc::ThreadActivity_Paused) {
bool thread_is_pinned = false; ThreadQueueImplForKThreadSetProperty wait_queue(std::addressof(m_pinned_waiter_list));
bool thread_is_current; bool thread_is_current;
do { do {
/* Lock the scheduler. */ /* Lock the scheduler. */
@ -903,23 +899,20 @@ namespace ams::kern {
/* Don't do any further management if our termination has been requested. */ /* Don't do any further management if our termination has been requested. */
R_SUCCEED_IF(this->IsTerminationRequested()); R_SUCCEED_IF(this->IsTerminationRequested());
/* By default, treat the thread as not current. */
thread_is_current = false;
/* Check whether the thread is pinned. */ /* Check whether the thread is pinned. */
if (this->GetStackParameters().is_pinned) { if (this->GetStackParameters().is_pinned) {
/* Verify that the current thread isn't terminating. */ /* Verify that the current thread isn't terminating. */
R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested()); R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
/* Note that the thread was pinned and not current. */
thread_is_pinned = true;
thread_is_current = false;
/* Wait until the thread isn't pinned any more. */ /* Wait until the thread isn't pinned any more. */
m_pinned_waiter_list.push_back(GetCurrentThread()); m_pinned_waiter_list.push_back(GetCurrentThread());
GetCurrentThread().SetState(ThreadState_Waiting); GetCurrentThread().BeginWait(std::addressof(wait_queue));
} else { } else {
/* Check if the thread is currently running. */ /* Check if the thread is currently running. */
/* If it is, we'll need to retry. */ /* If it is, we'll need to retry. */
thread_is_current = false;
for (auto i = 0; i < static_cast<s32>(cpu::NumCores); ++i) { for (auto i = 0; i < static_cast<s32>(cpu::NumCores); ++i) {
if (Kernel::GetScheduler(i).GetSchedulerCurrentThread() == this) { if (Kernel::GetScheduler(i).GetSchedulerCurrentThread() == this) {
thread_is_current = true; thread_is_current = true;
@ -928,15 +921,6 @@ namespace ams::kern {
} }
} }
} while (thread_is_current); } while (thread_is_current);
/* If the thread was pinned, it no longer is, and we should remove the current thread from our waiter list. */
if (thread_is_pinned) {
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Remove from the list. */
m_pinned_waiter_list.erase(m_pinned_waiter_list.iterator_to(GetCurrentThread()));
}
} }
return ResultSuccess(); return ResultSuccess();
@ -1241,8 +1225,9 @@ namespace ams::kern {
} }
/* Wake up the thread. */ /* Wake up the thread. */
this->SetSyncedObject(nullptr, svc::ResultTerminationRequested()); if (this->GetState() == ThreadState_Waiting) {
this->Wakeup(); m_wait_queue->CancelWait(this, svc::ResultTerminationRequested(), true);
}
} }
return this->GetState(); return this->GetState();
@ -1254,6 +1239,7 @@ namespace ams::kern {
MESOSPHERE_ASSERT(this == GetCurrentThreadPointer()); MESOSPHERE_ASSERT(this == GetCurrentThreadPointer());
MESOSPHERE_ASSERT(timeout > 0); MESOSPHERE_ASSERT(timeout > 0);
ThreadQueueImplForKThreadSleep wait_queue;
KHardwareTimer *timer; KHardwareTimer *timer;
{ {
/* Setup the scheduling lock and sleep. */ /* Setup the scheduling lock and sleep. */
@ -1265,18 +1251,58 @@ namespace ams::kern {
return svc::ResultTerminationRequested(); return svc::ResultTerminationRequested();
} }
/* Mark the thread as waiting. */ /* Wait for the sleep to end. */
this->SetState(KThread::ThreadState_Waiting); wait_queue.SetHardwareTimer(timer);
this->BeginWait(std::addressof(wait_queue));
} }
/* The lock/sleep is done. */
/* Cancel the timer. */
timer->CancelTask(this);
return ResultSuccess(); return ResultSuccess();
} }
void KThread::BeginWait(KThreadQueue *queue) {
/* Set our state as waiting. */
this->SetState(ThreadState_Waiting);
/* Set our wait queue. */
m_wait_queue = queue;
}
void KThread::NotifyAvailable(KSynchronizationObject *signaled_object, Result wait_result) {
MESOSPHERE_ASSERT_THIS();
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* If we're waiting, notify our queue that we're available. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->NotifyAvailable(this, signaled_object, wait_result);
}
}
void KThread::EndWait(Result wait_result) {
MESOSPHERE_ASSERT_THIS();
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* If we're waiting, notify our queue that we're available. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->EndWait(this, wait_result);
}
}
void KThread::CancelWait(Result wait_result, bool cancel_timer_task) {
MESOSPHERE_ASSERT_THIS();
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* If we're waiting, notify our queue that we're available. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->CancelWait(this, wait_result, cancel_timer_task);
}
}
void KThread::SetState(ThreadState state) { void KThread::SetState(ThreadState state) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();

62
libraries/libmesosphere/source/kern_k_thread_queue.cpp Normal file
View file

@ -0,0 +1,62 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern {
void KThreadQueue::NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result) {
MESOSPHERE_UNUSED(waiting_thread, signaled_object, wait_result);
MESOSPHERE_PANIC("KThreadQueue::NotifyAvailable\n");
}
void KThreadQueue::EndWait(KThread *waiting_thread, Result wait_result) {
/* Set the thread's wait result. */
waiting_thread->SetWaitResult(wait_result);
/* Set the thread as runnable. */
waiting_thread->SetState(KThread::ThreadState_Runnable);
/* Clear the thread's wait queue. */
waiting_thread->ClearWaitQueue();
/* Cancel the thread task. */
if (m_hardware_timer != nullptr) {
m_hardware_timer->CancelTask(waiting_thread);
}
}
void KThreadQueue::CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) {
/* Set the thread's wait result. */
waiting_thread->SetWaitResult(wait_result);
/* Set the thread as runnable. */
waiting_thread->SetState(KThread::ThreadState_Runnable);
/* Clear the thread's wait queue. */
waiting_thread->ClearWaitQueue();
/* Cancel the thread task. */
if (cancel_timer_task && m_hardware_timer != nullptr) {
m_hardware_timer->CancelTask(waiting_thread);
}
}
void KThreadQueueWithoutEndWait::EndWait(KThread *waiting_thread, Result wait_result) {
MESOSPHERE_UNUSED(waiting_thread, wait_result);
MESOSPHERE_PANIC("KThreadQueueWithoutEndWait::EndWait\n");
}
}

100
libraries/libmesosphere/source/kern_k_wait_object.cpp
View file

@ -17,73 +17,83 @@
namespace ams::kern { namespace ams::kern {
void KWaitObject::OnTimer() { namespace {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
/* Wake up all the waiting threads. */ class ThreadQueueImplForKWaitObjectSynchronize final : public KThreadQueueWithoutEndWait {
for (KThread &thread : m_wait_list) { private:
thread.Wakeup(); KThread::WaiterList *m_wait_list;
KThread **m_thread;
public:
constexpr ThreadQueueImplForKWaitObjectSynchronize(KThread::WaiterList *wl, KThread **t) : KThreadQueueWithoutEndWait(), m_wait_list(wl), m_thread(t) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread from the wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* If the result was a timeout and the thread is our wait object thread, cancel recursively. */
if (svc::ResultTimedOut::Includes(wait_result) && waiting_thread == *m_thread) {
for (auto &thread : *m_wait_list) {
thread.CancelWait(svc::ResultTimedOut(), false);
} }
} }
/* If the thread is our wait object thread, clear it. */
if (*m_thread == waiting_thread) {
*m_thread = nullptr;
}
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
Result KWaitObject::Synchronize(s64 timeout) { Result KWaitObject::Synchronize(s64 timeout) {
/* Perform the wait. */ /* Perform the wait. */
KHardwareTimer *timer = nullptr; KHardwareTimer *timer;
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
ThreadQueueImplForKWaitObjectSynchronize wait_queue(std::addressof(m_wait_list), std::addressof(m_next_thread));
{ {
KScopedSchedulerLock sl; KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
/* Check that the thread isn't terminating. */ /* Check that the thread isn't terminating. */
R_UNLESS(!cur_thread->IsTerminationRequested(), svc::ResultTerminationRequested()); if (cur_thread->IsTerminationRequested()) {
slp.CancelSleep();
/* Verify that nothing else is already waiting on the object. */ return svc::ResultTerminationRequested();
if (timeout > 0) {
R_UNLESS(!m_timer_used, svc::ResultBusy());
} }
/* Check that we're not already in use. */ /* Handle the case where timeout is non-negative/infinite. */
if (timeout >= 0) { if (timeout >= 0) {
/* Verify the timer isn't already in use. */ /* Check if we're already waiting. */
R_UNLESS(!m_timer_used, svc::ResultBusy()); if (m_next_thread != nullptr) {
} slp.CancelSleep();
return svc::ResultBusy();
/* If we need to, register our timeout. */
if (timeout > 0) {
/* Mark that we're using the timer. */
m_timer_used = true;
/* Use the timer. */
timer = std::addressof(Kernel::GetHardwareTimer());
timer->RegisterAbsoluteTask(this, timeout);
} }
/* If timeout is zero, handle the special case by canceling all waiting threads. */
if (timeout == 0) { if (timeout == 0) {
/* If we're timed out immediately, just wake up the thread. */ for (auto &thread : m_wait_list) {
this->OnTimer(); thread.CancelWait(svc::ResultTimedOut(), false);
} else { }
/* Otherwise, sleep until the timeout occurs. */
m_wait_list.push_back(GetCurrentThread()); slp.CancelSleep();
cur_thread->SetState(KThread::ThreadState_Waiting); return ResultSuccess();
cur_thread->SetSyncedObject(nullptr, svc::ResultTimedOut());
} }
} }
/* Cleanup as necessary. */ /* If the timeout isn't infinite, register it as our next timeout. */
{
KScopedSchedulerLock sl;
/* Remove from the timer. */
if (timeout > 0) { if (timeout > 0) {
MESOSPHERE_ASSERT(m_timer_used); wait_queue.SetHardwareTimer(timer);
MESOSPHERE_ASSERT(timer != nullptr); m_next_thread = cur_thread;
timer->CancelTask(this);
m_timer_used = false;
} }
/* Remove the thread from our queue. */ /* Add the current thread to our wait list. */
if (timeout != 0) { m_wait_list.push_back(*cur_thread);
m_wait_list.erase(m_wait_list.iterator_to(GetCurrentThread()));
} /* Wait until the timeout occurs. */
cur_thread->BeginWait(std::addressof(wait_queue));
} }
return ResultSuccess(); return ResultSuccess();

59
libraries/libmesosphere/source/kern_k_worker_task_manager.cpp
View file

@ -17,22 +17,46 @@
namespace ams::kern { namespace ams::kern {
namespace {
class ThreadQueueImplForKWorkerTaskManager final : public KThreadQueue {
private:
KThread **m_waiting_thread;
public:
constexpr ThreadQueueImplForKWorkerTaskManager(KThread **t) : KThreadQueue(), m_waiting_thread(t) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Clear our waiting thread. */
*m_waiting_thread = nullptr;
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
MESOSPHERE_UNUSED(waiting_thread, wait_result, cancel_timer_task);
MESOSPHERE_PANIC("ThreadQueueImplForKWorkerTaskManager::CancelWait\n");
}
};
}
void KWorkerTaskManager::Initialize(s32 priority) { void KWorkerTaskManager::Initialize(s32 priority) {
/* Reserve a thread from the system limit. */ /* Reserve a thread from the system limit. */
MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_ThreadCountMax, 1)); MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_ThreadCountMax, 1));
/* Create a new thread. */ /* Create a new thread. */
m_thread = KThread::Create(); KThread *thread = KThread::Create();
MESOSPHERE_ABORT_UNLESS(m_thread != nullptr); MESOSPHERE_ABORT_UNLESS(thread != nullptr);
/* Launch the new thread. */ /* Launch the new thread. */
MESOSPHERE_R_ABORT_UNLESS(KThread::InitializeKernelThread(m_thread, ThreadFunction, reinterpret_cast<uintptr_t>(this), priority, cpu::NumCores - 1)); MESOSPHERE_R_ABORT_UNLESS(KThread::InitializeKernelThread(thread, ThreadFunction, reinterpret_cast<uintptr_t>(this), priority, cpu::NumCores - 1));
/* Register the new thread. */ /* Register the new thread. */
KThread::Register(m_thread); KThread::Register(thread);
/* Run the thread. */ /* Run the thread. */
m_thread->Run(); thread->Run();
} }
void KWorkerTaskManager::AddTask(WorkerType type, KWorkerTask *task) { void KWorkerTaskManager::AddTask(WorkerType type, KWorkerTask *task) {
@ -45,36 +69,40 @@ namespace ams::kern {
} }
void KWorkerTaskManager::ThreadFunctionImpl() { void KWorkerTaskManager::ThreadFunctionImpl() {
/* Create wait queue. */
ThreadQueueImplForKWorkerTaskManager wait_queue(std::addressof(m_waiting_thread));
while (true) { while (true) {
KWorkerTask *task = nullptr; KWorkerTask *task;
/* Get a worker task. */ /* Get a worker task. */
{ {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
task = this->GetTask(); task = this->GetTask();
if (task == nullptr) { if (task == nullptr) {
/* If there's nothing to do, set ourselves as waiting. */ /* Wait to have a task. */
m_active = false; m_waiting_thread = GetCurrentThreadPointer();
m_thread->SetState(KThread::ThreadState_Waiting); GetCurrentThread().BeginWait(std::addressof(wait_queue));
continue; continue;
} }
m_active = true;
} }
/* Do the task. */ /* Do the task. */
task->DoWorkerTask(); task->DoWorkerTask();
/* Destroy any objects we may need to close. */ /* Destroy any objects we may need to close. */
m_thread->DestroyClosedObjects(); GetCurrentThread().DestroyClosedObjects();
} }
} }
KWorkerTask *KWorkerTaskManager::GetTask() { KWorkerTask *KWorkerTaskManager::GetTask() {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
KWorkerTask *next = m_head_task; KWorkerTask *next = m_head_task;
if (next) {
if (next != nullptr) {
/* Advance the list. */ /* Advance the list. */
if (m_head_task == m_tail_task) { if (m_head_task == m_tail_task) {
m_head_task = nullptr; m_head_task = nullptr;
@ -86,6 +114,7 @@ namespace ams::kern {
/* Clear the next task's next. */ /* Clear the next task's next. */
next->SetNextTask(nullptr); next->SetNextTask(nullptr);
} }
return next; return next;
} }
@ -102,8 +131,8 @@ namespace ams::kern {
m_tail_task = task; m_tail_task = task;
/* Make ourselves active if we need to. */ /* Make ourselves active if we need to. */
if (!m_active) { if (m_waiting_thread != nullptr) {
m_thread->SetState(KThread::ThreadState_Runnable); m_waiting_thread->EndWait(ResultSuccess());
} }
} }
} }

9
libraries/libmesosphere/source/svc/kern_svc_ipc.cpp
View file

@ -229,11 +229,14 @@ namespace ams::kern::svc {
/* Send the request. */ /* Send the request. */
MESOSPHERE_ASSERT(message != 0); MESOSPHERE_ASSERT(message != 0);
R_TRY(SendAsyncRequestWithUserBufferImpl(out_event_handle, message, buffer_size, session_handle)); const Result result = SendAsyncRequestWithUserBufferImpl(out_event_handle, message, buffer_size, session_handle);
/* We sent the request successfully. */ /* If the request succeeds (or the thread is terminating), don't unlock the user buffer. */
if (R_SUCCEEDED(result) || svc::ResultTerminationRequested::Includes(result)) {
unlock_guard.Cancel(); unlock_guard.Cancel();
return ResultSuccess(); }
return result;
} }
ALWAYS_INLINE Result ReplyAndReceive(int32_t *out_index, KUserPointer<const ams::svc::Handle *> handles, int32_t num_handles, ams::svc::Handle reply_target, int64_t timeout_ns) { ALWAYS_INLINE Result ReplyAndReceive(int32_t *out_index, KUserPointer<const ams::svc::Handle *> handles, int32_t num_handles, ams::svc::Handle reply_target, int64_t timeout_ns) {