1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-01-30 12:53:52 +00:00

kern: unify all waiting semantics to use single api

This commit is contained in:
Michael Scire 2021-09-19 10:11:56 -07:00 committed by SciresM
parent f6fb5f2c8d
commit 90732ff311
22 changed files with 904 additions and 683 deletions

View file

@ -27,59 +27,9 @@ namespace ams::kern {
KThread::WaiterList m_wait_list; KThread::WaiterList m_wait_list;
public: public:
constexpr ALWAYS_INLINE KLightConditionVariable() : m_wait_list() { /* ... */ } constexpr ALWAYS_INLINE KLightConditionVariable() : m_wait_list() { /* ... */ }
private:
void WaitImpl(KLightLock *lock, s64 timeout, bool allow_terminating_thread) {
KThread *owner = GetCurrentThreadPointer();
KHardwareTimer *timer;
/* Sleep the thread. */
{
KScopedSchedulerLockAndSleep lk(&timer, owner, timeout);
if (!allow_terminating_thread && owner->IsTerminationRequested()) {
lk.CancelSleep();
return;
}
lock->Unlock();
/* Set the thread as waiting. */
GetCurrentThread().SetState(KThread::ThreadState_Waiting);
/* Add the thread to the queue. */
m_wait_list.push_back(GetCurrentThread());
}
/* Remove the thread from the wait list. */
{
KScopedSchedulerLock sl;
m_wait_list.erase(m_wait_list.iterator_to(GetCurrentThread()));
}
/* Cancel the task that the sleep setup. */
if (timer != nullptr) {
timer->CancelTask(owner);
}
/* Re-acquire the lock. */
lock->Lock();
}
public: public:
void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true) { void Wait(KLightLock *lock, s64 timeout = -1ll, bool allow_terminating_thread = true);
this->WaitImpl(lock, timeout, allow_terminating_thread); void Broadcast();
}
void Broadcast() {
KScopedSchedulerLock lk;
/* Signal all threads. */
for (auto &thread : m_wait_list) {
thread.SetState(KThread::ThreadState_Runnable);
}
}
}; };
} }

View file

@ -27,12 +27,12 @@ namespace ams::kern {
MESOSPHERE_AUTOOBJECT_TRAITS(KLightServerSession, KAutoObject); MESOSPHERE_AUTOOBJECT_TRAITS(KLightServerSession, KAutoObject);
private: private:
KLightSession *m_parent; KLightSession *m_parent;
KThreadQueue m_request_queue; KThread::WaiterList m_request_list;
KThreadQueue m_server_queue;
KThread *m_current_request; KThread *m_current_request;
u64 m_server_thread_id;
KThread *m_server_thread; KThread *m_server_thread;
public: public:
constexpr KLightServerSession() : m_parent(), m_request_queue(), m_server_queue(), m_current_request(), m_server_thread() { /* ... */ } constexpr KLightServerSession() : m_parent(), m_request_list(), m_current_request(), m_server_thread_id(), m_server_thread() { /* ... */ }
void Initialize(KLightSession *parent) { void Initialize(KLightSession *parent) {
/* Set member variables. */ /* Set member variables. */

View file

@ -45,7 +45,39 @@ namespace ams::kern {
public: public:
virtual void Finalize() override; virtual void Finalize() override;
virtual bool IsSignaled() const { AMS_INFINITE_LOOP(); } virtual bool IsSignaled() const { AMS_INFINITE_LOOP(); }
virtual void DumpWaiters();
void DumpWaiters();
ALWAYS_INLINE void LinkNode(ThreadListNode *node) {
/* Link the node to the list. */
if (m_thread_list_tail == nullptr) {
m_thread_list_head = node;
} else {
m_thread_list_tail->next = node;
}
m_thread_list_tail = node;
}
ALWAYS_INLINE void UnlinkNode(ThreadListNode *node) {
/* Unlink the node from the list. */
ThreadListNode *prev_ptr = reinterpret_cast<ThreadListNode *>(std::addressof(m_thread_list_head));
ThreadListNode *prev_val = nullptr;
ThreadListNode *prev, *tail_prev;
do {
prev = prev_ptr;
prev_ptr = prev_ptr->next;
tail_prev = prev_val;
prev_val = prev_ptr;
} while (prev_ptr != node);
if (m_thread_list_tail == node) {
m_thread_list_tail = tail_prev;
}
prev->next = node->next;
}
}; };
} }

View file

@ -37,6 +37,7 @@ namespace ams::kern {
friend class KProcess; friend class KProcess;
friend class KConditionVariable; friend class KConditionVariable;
friend class KAddressArbiter; friend class KAddressArbiter;
friend class KThreadQueue;
public: public:
static constexpr s32 MainThreadPriority = 1; static constexpr s32 MainThreadPriority = 1;
static constexpr s32 IdleThreadPriority = 64; static constexpr s32 IdleThreadPriority = 64;
@ -191,7 +192,6 @@ namespace ams::kern {
KAffinityMask m_physical_affinity_mask{}; KAffinityMask m_physical_affinity_mask{};
u64 m_thread_id{}; u64 m_thread_id{};
std::atomic<s64> m_cpu_time{}; std::atomic<s64> m_cpu_time{};
KSynchronizationObject *m_synced_object{};
KProcessAddress m_address_key{}; KProcessAddress m_address_key{};
KProcess *m_parent{}; KProcess *m_parent{};
void *m_kernel_stack_top{}; void *m_kernel_stack_top{};
@ -204,9 +204,7 @@ namespace ams::kern {
s64 m_last_scheduled_tick{}; s64 m_last_scheduled_tick{};
QueueEntry m_per_core_priority_queue_entry[cpu::NumCores]{}; QueueEntry m_per_core_priority_queue_entry[cpu::NumCores]{};
KLightLock *m_waiting_lock{}; KLightLock *m_waiting_lock{};
KThreadQueue *m_wait_queue{};
KThreadQueue *m_sleeping_queue{};
WaiterList m_waiter_list{}; WaiterList m_waiter_list{};
WaiterList m_pinned_waiter_list{}; WaiterList m_pinned_waiter_list{};
KThread *m_lock_owner{}; KThread *m_lock_owner{};
@ -215,6 +213,7 @@ namespace ams::kern {
u32 m_address_key_value{}; u32 m_address_key_value{};
u32 m_suspend_request_flags{}; u32 m_suspend_request_flags{};
u32 m_suspend_allowed_flags{}; u32 m_suspend_allowed_flags{};
s32 m_synced_index{};
Result m_wait_result; Result m_wait_result;
Result m_debug_exception_result; Result m_debug_exception_result;
s32 m_base_priority{}; s32 m_base_priority{};
@ -374,6 +373,8 @@ namespace ams::kern {
void FinishTermination(); void FinishTermination();
void IncreaseBasePriority(s32 priority); void IncreaseBasePriority(s32 priority);
NOINLINE void SetState(ThreadState state);
public: public:
constexpr u64 GetThreadId() const { return m_thread_id; } constexpr u64 GetThreadId() const { return m_thread_id; }
@ -390,7 +391,6 @@ namespace ams::kern {
constexpr ThreadState GetState() const { return static_cast<ThreadState>(m_thread_state & ThreadState_Mask); } constexpr ThreadState GetState() const { return static_cast<ThreadState>(m_thread_state & ThreadState_Mask); }
constexpr ThreadState GetRawState() const { return m_thread_state; } constexpr ThreadState GetRawState() const { return m_thread_state; }
NOINLINE void SetState(ThreadState state);
NOINLINE KThreadContext *GetContextForSchedulerLoop(); NOINLINE KThreadContext *GetContextForSchedulerLoop();
@ -442,8 +442,6 @@ namespace ams::kern {
constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return m_per_core_priority_queue_entry[core]; } constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return m_per_core_priority_queue_entry[core]; }
constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return m_per_core_priority_queue_entry[core]; } constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return m_per_core_priority_queue_entry[core]; }
constexpr void SetSleepingQueue(KThreadQueue *q) { m_sleeping_queue = q; }
constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return m_condvar_tree; } constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return m_condvar_tree; }
constexpr s32 GetNumKernelWaiters() const { return m_num_kernel_waiters; } constexpr s32 GetNumKernelWaiters() const { return m_num_kernel_waiters; }
@ -460,29 +458,22 @@ namespace ams::kern {
constexpr void SetLockOwner(KThread *owner) { m_lock_owner = owner; } constexpr void SetLockOwner(KThread *owner) { m_lock_owner = owner; }
constexpr KThread *GetLockOwner() const { return m_lock_owner; } constexpr KThread *GetLockOwner() const { return m_lock_owner; }
constexpr void SetSyncedObject(KSynchronizationObject *obj, Result wait_res) { constexpr void ClearWaitQueue() { m_wait_queue = nullptr; }
MESOSPHERE_ASSERT_THIS();
m_synced_object = obj; void BeginWait(KThreadQueue *queue);
m_wait_result = wait_res; void NotifyAvailable(KSynchronizationObject *signaled_object, Result wait_result);
} void EndWait(Result wait_result);
void CancelWait(Result wait_result, bool cancel_timer_task);
constexpr Result GetWaitResult(KSynchronizationObject **out) const { constexpr void SetSyncedIndex(s32 index) { m_synced_index = index; }
MESOSPHERE_ASSERT_THIS(); constexpr s32 GetSyncedIndex() const { return m_synced_index; }
*out = m_synced_object; constexpr void SetWaitResult(Result wait_res) { m_wait_result = wait_res; }
return m_wait_result; constexpr Result GetWaitResult() const { return m_wait_result; }
}
constexpr void SetDebugExceptionResult(Result result) { constexpr void SetDebugExceptionResult(Result result) { m_debug_exception_result = result; }
MESOSPHERE_ASSERT_THIS();
m_debug_exception_result = result;
}
constexpr Result GetDebugExceptionResult() const { constexpr Result GetDebugExceptionResult() const { return m_debug_exception_result; }
MESOSPHERE_ASSERT_THIS();
return m_debug_exception_result;
}
void WaitCancel(); void WaitCancel();
@ -585,8 +576,6 @@ namespace ams::kern {
} }
} }
void Wakeup();
void SetBasePriority(s32 priority); void SetBasePriority(s32 priority);
Result SetPriorityToIdle(); Result SetPriorityToIdle();

View file

@ -16,69 +16,28 @@
#pragma once #pragma once
#include <mesosphere/kern_common.hpp> #include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_thread.hpp> #include <mesosphere/kern_k_thread.hpp>
#include <mesosphere/kern_select_hardware_timer.hpp>
namespace ams::kern { namespace ams::kern {
class KThreadQueue { class KThreadQueue {
private: private:
KThread::WaiterList m_wait_list; KHardwareTimer *m_hardware_timer;
public: public:
constexpr ALWAYS_INLINE KThreadQueue() : m_wait_list() { /* ... */ } constexpr ALWAYS_INLINE KThreadQueue() : m_hardware_timer(nullptr) { /* ... */ }
bool IsEmpty() const { return m_wait_list.empty(); } constexpr void SetHardwareTimer(KHardwareTimer *timer) { m_hardware_timer = timer; }
KThread::WaiterList::iterator begin() { return m_wait_list.begin(); } virtual void NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result);
KThread::WaiterList::iterator end() { return m_wait_list.end(); } virtual void EndWait(KThread *waiting_thread, Result wait_result);
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task);
};
bool SleepThread(KThread *t) { class KThreadQueueWithoutEndWait : public KThreadQueue {
KScopedSchedulerLock sl; public:
constexpr ALWAYS_INLINE KThreadQueueWithoutEndWait() : KThreadQueue() { /* ... */ }
/* If the thread needs terminating, don't enqueue it. */ virtual void EndWait(KThread *waiting_thread, Result wait_result) override final;
if (t->IsTerminationRequested()) {
return false;
}
/* Set the thread's queue and mark it as waiting. */
t->SetSleepingQueue(this);
t->SetState(KThread::ThreadState_Waiting);
/* Add the thread to the queue. */
m_wait_list.push_back(*t);
return true;
}
void WakeupThread(KThread *t) {
KScopedSchedulerLock sl;
/* Remove the thread from the queue. */
m_wait_list.erase(m_wait_list.iterator_to(*t));
/* Mark the thread as no longer sleeping. */
t->SetState(KThread::ThreadState_Runnable);
t->SetSleepingQueue(nullptr);
}
KThread *WakeupFrontThread() {
KScopedSchedulerLock sl;
if (m_wait_list.empty()) {
return nullptr;
} else {
/* Remove the thread from the queue. */
auto it = m_wait_list.begin();
KThread *thread = std::addressof(*it);
m_wait_list.erase(it);
MESOSPHERE_ASSERT(thread->GetState() == KThread::ThreadState_Waiting);
/* Mark the thread as no longer sleeping. */
thread->SetState(KThread::ThreadState_Runnable);
thread->SetSleepingQueue(nullptr);
return thread;
}
}
}; };
} }

View file

@ -20,14 +20,13 @@
namespace ams::kern { namespace ams::kern {
class KWaitObject : public KTimerTask { class KWaitObject {
private: private:
KThread::WaiterList m_wait_list; KThread::WaiterList m_wait_list;
bool m_timer_used; KThread *m_next_thread;
public: public:
constexpr KWaitObject() : m_wait_list(), m_timer_used() { /* ... */ } constexpr KWaitObject() : m_wait_list(), m_next_thread() { /* ... */ }
virtual void OnTimer() override;
Result Synchronize(s64 timeout); Result Synchronize(s64 timeout);
}; };

View file

@ -32,8 +32,7 @@ namespace ams::kern {
private: private:
KWorkerTask *m_head_task; KWorkerTask *m_head_task;
KWorkerTask *m_tail_task; KWorkerTask *m_tail_task;
KThread *m_thread; KThread *m_waiting_thread;
bool m_active;
private: private:
static void ThreadFunction(uintptr_t arg); static void ThreadFunction(uintptr_t arg);
void ThreadFunctionImpl(); void ThreadFunctionImpl();
@ -41,7 +40,7 @@ namespace ams::kern {
KWorkerTask *GetTask(); KWorkerTask *GetTask();
void AddTask(KWorkerTask *task); void AddTask(KWorkerTask *task);
public: public:
constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_thread(), m_active() { /* ... */ } constexpr KWorkerTaskManager() : m_head_task(), m_tail_task(), m_waiting_thread() { /* ... */ }
NOINLINE void Initialize(s32 priority); NOINLINE void Initialize(s32 priority);
static void AddTask(WorkerType type, KWorkerTask *task); static void AddTask(WorkerType type, KWorkerTask *task);

View file

@ -43,6 +43,24 @@ namespace ams::kern {
return UserspaceAccess::UpdateIfEqualAtomic(out, GetPointer<s32>(address), value, new_value); return UserspaceAccess::UpdateIfEqualAtomic(out, GetPointer<s32>(address), value, new_value);
} }
class ThreadQueueImplForKAddressArbiter final : public KThreadQueue {
private:
KAddressArbiter::ThreadTree *m_tree;
public:
constexpr ThreadQueueImplForKAddressArbiter(KAddressArbiter::ThreadTree *t) : KThreadQueue(), m_tree(t) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* If the thread is waiting on an address arbiter, remove it from the tree. */
if (waiting_thread->IsWaitingForAddressArbiter()) {
m_tree->erase(m_tree->iterator_to(*waiting_thread));
waiting_thread->ClearAddressArbiter();
}
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
Result KAddressArbiter::Signal(uintptr_t addr, s32 count) { Result KAddressArbiter::Signal(uintptr_t addr, s32 count) {
@ -53,14 +71,14 @@ namespace ams::kern {
auto it = m_tree.nfind_key({ addr, -1 }); auto it = m_tree.nfind_key({ addr, -1 });
while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
/* End the thread's wait. */
KThread *target_thread = std::addressof(*it); KThread *target_thread = std::addressof(*it);
target_thread->SetSyncedObject(nullptr, ResultSuccess()); target_thread->EndWait(ResultSuccess());
AMS_ASSERT(target_thread->IsWaitingForAddressArbiter()); MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
target_thread->Wakeup(); target_thread->ClearAddressArbiter();
it = m_tree.erase(it); it = m_tree.erase(it);
target_thread->ClearAddressArbiter();
++num_waiters; ++num_waiters;
} }
} }
@ -80,14 +98,14 @@ namespace ams::kern {
auto it = m_tree.nfind_key({ addr, -1 }); auto it = m_tree.nfind_key({ addr, -1 });
while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
/* End the thread's wait. */
KThread *target_thread = std::addressof(*it); KThread *target_thread = std::addressof(*it);
target_thread->SetSyncedObject(nullptr, ResultSuccess()); target_thread->EndWait(ResultSuccess());
AMS_ASSERT(target_thread->IsWaitingForAddressArbiter()); MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
target_thread->Wakeup(); target_thread->ClearAddressArbiter();
it = m_tree.erase(it); it = m_tree.erase(it);
target_thread->ClearAddressArbiter();
++num_waiters; ++num_waiters;
} }
} }
@ -142,14 +160,14 @@ namespace ams::kern {
R_UNLESS(user_value == value, svc::ResultInvalidState()); R_UNLESS(user_value == value, svc::ResultInvalidState());
while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) { while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) && (it->GetAddressArbiterKey() == addr)) {
/* End the thread's wait. */
KThread *target_thread = std::addressof(*it); KThread *target_thread = std::addressof(*it);
target_thread->SetSyncedObject(nullptr, ResultSuccess()); target_thread->EndWait(ResultSuccess());
AMS_ASSERT(target_thread->IsWaitingForAddressArbiter()); MESOSPHERE_ASSERT(target_thread->IsWaitingForAddressArbiter());
target_thread->Wakeup(); target_thread->ClearAddressArbiter();
it = m_tree.erase(it); it = m_tree.erase(it);
target_thread->ClearAddressArbiter();
++num_waiters; ++num_waiters;
} }
} }
@ -160,6 +178,7 @@ namespace ams::kern {
/* Prepare to wait. */ /* Prepare to wait. */
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKAddressArbiter wait_queue(std::addressof(m_tree));
{ {
KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
@ -170,9 +189,6 @@ namespace ams::kern {
return svc::ResultTerminationRequested(); return svc::ResultTerminationRequested();
} }
/* Set the synced object. */
cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
/* Read the value from userspace. */ /* Read the value from userspace. */
s32 user_value; s32 user_value;
bool succeeded; bool succeeded;
@ -202,33 +218,21 @@ namespace ams::kern {
/* Set the arbiter. */ /* Set the arbiter. */
cur_thread->SetAddressArbiter(std::addressof(m_tree), addr); cur_thread->SetAddressArbiter(std::addressof(m_tree), addr);
m_tree.insert(*cur_thread); m_tree.insert(*cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
/* Wait for the thread to finish. */
wait_queue.SetHardwareTimer(timer);
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* Cancel the timer wait. */ /* Get the wait result. */
if (timer != nullptr) { return cur_thread->GetWaitResult();
timer->CancelTask(cur_thread);
}
/* Remove from the address arbiter. */
{
KScopedSchedulerLock sl;
if (cur_thread->IsWaitingForAddressArbiter()) {
m_tree.erase(m_tree.iterator_to(*cur_thread));
cur_thread->ClearAddressArbiter();
}
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
Result KAddressArbiter::WaitIfEqual(uintptr_t addr, s32 value, s64 timeout) { Result KAddressArbiter::WaitIfEqual(uintptr_t addr, s32 value, s64 timeout) {
/* Prepare to wait. */ /* Prepare to wait. */
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKAddressArbiter wait_queue(std::addressof(m_tree));
{ {
KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
@ -239,9 +243,6 @@ namespace ams::kern {
return svc::ResultTerminationRequested(); return svc::ResultTerminationRequested();
} }
/* Set the synced object. */
cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
/* Read the value from userspace. */ /* Read the value from userspace. */
s32 user_value; s32 user_value;
if (!ReadFromUser(std::addressof(user_value), addr)) { if (!ReadFromUser(std::addressof(user_value), addr)) {
@ -264,27 +265,14 @@ namespace ams::kern {
/* Set the arbiter. */ /* Set the arbiter. */
cur_thread->SetAddressArbiter(std::addressof(m_tree), addr); cur_thread->SetAddressArbiter(std::addressof(m_tree), addr);
m_tree.insert(*cur_thread); m_tree.insert(*cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
/* Wait for the thread to finish. */
wait_queue.SetHardwareTimer(timer);
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* Cancel the timer wait. */ /* Get the wait result. */
if (timer != nullptr) { return cur_thread->GetWaitResult();
timer->CancelTask(cur_thread);
}
/* Remove from the address arbiter. */
{
KScopedSchedulerLock sl;
if (cur_thread->IsWaitingForAddressArbiter()) {
m_tree.erase(m_tree.iterator_to(*cur_thread));
cur_thread->ClearAddressArbiter();
}
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
} }

View file

@ -40,17 +40,7 @@ namespace ams::kern {
request->Initialize(nullptr, address, size); request->Initialize(nullptr, address, size);
/* Send the request. */ /* Send the request. */
{ return m_parent->OnRequest(request);
KScopedSchedulerLock sl;
GetCurrentThread().SetSyncedObject(nullptr, ResultSuccess());
R_TRY(m_parent->OnRequest(request));
}
/* Get the result. */
KSynchronizationObject *dummy;
return GetCurrentThread().GetWaitResult(std::addressof(dummy));
} }
Result KClientSession::SendAsyncRequest(KEvent *event, uintptr_t address, size_t size) { Result KClientSession::SendAsyncRequest(KEvent *event, uintptr_t address, size_t size) {
@ -65,13 +55,7 @@ namespace ams::kern {
request->Initialize(event, address, size); request->Initialize(event, address, size);
/* Send the request. */ /* Send the request. */
{ return m_parent->OnRequest(request);
KScopedSchedulerLock sl;
R_TRY(m_parent->OnRequest(request));
}
return ResultSuccess();
} }
} }

View file

@ -31,10 +31,46 @@ namespace ams::kern {
return UserspaceAccess::UpdateLockAtomic(out, GetPointer<u32>(address), if_zero, new_orr_mask); return UserspaceAccess::UpdateLockAtomic(out, GetPointer<u32>(address), if_zero, new_orr_mask);
} }
class ThreadQueueImplForKConditionVariableWaitForAddress final : public KThreadQueue {
public:
constexpr ThreadQueueImplForKConditionVariableWaitForAddress() : KThreadQueue() { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread as a waiter from its owner. */
waiting_thread->GetLockOwner()->RemoveWaiter(waiting_thread);
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
class ThreadQueueImplForKConditionVariableWaitConditionVariable final : public KThreadQueue {
private:
KConditionVariable::ThreadTree *m_tree;
public:
constexpr ThreadQueueImplForKConditionVariableWaitConditionVariable(KConditionVariable::ThreadTree *t) : KThreadQueue(), m_tree(t) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread as a waiter from its owner. */
if (KThread *owner = waiting_thread->GetLockOwner(); owner != nullptr) {
owner->RemoveWaiter(waiting_thread);
}
/* If the thread is waiting on a condvar, remove it from the tree. */
if (waiting_thread->IsWaitingForConditionVariable()) {
m_tree->erase(m_tree->iterator_to(*waiting_thread));
waiting_thread->ClearConditionVariable();
}
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
Result KConditionVariable::SignalToAddress(KProcessAddress addr) { Result KConditionVariable::SignalToAddress(KProcessAddress addr) {
KThread *owner_thread = std::addressof(GetCurrentThread()); KThread *owner_thread = GetCurrentThreadPointer();
/* Signal the address. */ /* Signal the address. */
{ {
@ -46,44 +82,43 @@ namespace ams::kern {
/* Determine the next tag. */ /* Determine the next tag. */
u32 next_value = 0; u32 next_value = 0;
if (next_owner_thread) { if (next_owner_thread != nullptr) {
next_value = next_owner_thread->GetAddressKeyValue(); next_value = next_owner_thread->GetAddressKeyValue();
if (num_waiters > 1) { if (num_waiters > 1) {
next_value |= ams::svc::HandleWaitMask; next_value |= ams::svc::HandleWaitMask;
} }
next_owner_thread->SetSyncedObject(nullptr, ResultSuccess());
next_owner_thread->Wakeup();
}
/* Write the value to userspace. */ /* Write the value to userspace. */
if (!WriteToUser(addr, std::addressof(next_value))) { Result result;
if (next_owner_thread) { if (AMS_LIKELY(WriteToUser(addr, std::addressof(next_value)))) {
next_owner_thread->SetSyncedObject(nullptr, svc::ResultInvalidCurrentMemory()); result = ResultSuccess();
} else {
result = svc::ResultInvalidCurrentMemory();
} }
return svc::ResultInvalidCurrentMemory(); /* Signal the next owner thread. */
} next_owner_thread->EndWait(result);
} return result;
} else {
/* Just write the value to userspace. */
R_UNLESS(WriteToUser(addr, std::addressof(next_value)), svc::ResultInvalidCurrentMemory());
return ResultSuccess(); return ResultSuccess();
} }
}
}
Result KConditionVariable::WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value) { Result KConditionVariable::WaitForAddress(ams::svc::Handle handle, KProcessAddress addr, u32 value) {
KThread *cur_thread = std::addressof(GetCurrentThread()); KThread *cur_thread = GetCurrentThreadPointer();
ThreadQueueImplForKConditionVariableWaitForAddress wait_queue;
/* Wait for the address. */ /* Wait for the address. */
{
KScopedAutoObject<KThread> owner_thread;
MESOSPHERE_ASSERT(owner_thread.IsNull());
{ {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
cur_thread->SetSyncedObject(nullptr, ResultSuccess());
/* Check if the thread should terminate. */ /* Check if the thread should terminate. */
R_UNLESS(!cur_thread->IsTerminationRequested(), svc::ResultTerminationRequested()); R_UNLESS(!cur_thread->IsTerminationRequested(), svc::ResultTerminationRequested());
{
/* Read the tag from userspace. */ /* Read the tag from userspace. */
u32 test_tag; u32 test_tag;
R_UNLESS(ReadFromUser(std::addressof(test_tag), addr), svc::ResultInvalidCurrentMemory()); R_UNLESS(ReadFromUser(std::addressof(test_tag), addr), svc::ResultInvalidCurrentMemory());
@ -92,29 +127,19 @@ namespace ams::kern {
R_SUCCEED_IF(test_tag != (handle | ams::svc::HandleWaitMask)); R_SUCCEED_IF(test_tag != (handle | ams::svc::HandleWaitMask));
/* Get the lock owner thread. */ /* Get the lock owner thread. */
owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(handle); KScopedAutoObject owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(handle);
R_UNLESS(owner_thread.IsNotNull(), svc::ResultInvalidHandle()); R_UNLESS(owner_thread.IsNotNull(), svc::ResultInvalidHandle());
/* Update the lock. */ /* Update the lock. */
cur_thread->SetAddressKey(addr, value); cur_thread->SetAddressKey(addr, value);
owner_thread->AddWaiter(cur_thread); owner_thread->AddWaiter(cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
}
}
MESOSPHERE_ASSERT(owner_thread.IsNotNull());
/* Remove the thread as a waiter from the lock owner. */ /* Begin waiting. */
{ cur_thread->BeginWait(std::addressof(wait_queue));
KScopedSchedulerLock sl;
if (KThread *mutex_owner = cur_thread->GetLockOwner(); mutex_owner != nullptr) {
mutex_owner->RemoveWaiter(cur_thread);
}
}
} }
/* Get the wait result. */ /* Get the wait result. */
KSynchronizationObject *dummy; return cur_thread->GetWaitResult();
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
void KConditionVariable::SignalImpl(KThread *thread) { void KConditionVariable::SignalImpl(KThread *thread) {
@ -139,8 +164,7 @@ namespace ams::kern {
if (AMS_LIKELY(can_access)) { if (AMS_LIKELY(can_access)) {
if (prev_tag == ams::svc::InvalidHandle) { if (prev_tag == ams::svc::InvalidHandle) {
/* If nobody held the lock previously, we're all good. */ /* If nobody held the lock previously, we're all good. */
thread->SetSyncedObject(nullptr, ResultSuccess()); thread->EndWait(ResultSuccess());
thread->Wakeup();
} else { } else {
/* Get the previous owner. */ /* Get the previous owner. */
KThread *owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(static_cast<ams::svc::Handle>(prev_tag & ~ams::svc::HandleWaitMask)) KThread *owner_thread = GetCurrentProcess().GetHandleTable().GetObjectWithoutPseudoHandle<KThread>(static_cast<ams::svc::Handle>(prev_tag & ~ams::svc::HandleWaitMask))
@ -151,14 +175,12 @@ namespace ams::kern {
owner_thread->Close(); owner_thread->Close();
} else { } else {
/* The lock was tagged with a thread that doesn't exist. */ /* The lock was tagged with a thread that doesn't exist. */
thread->SetSyncedObject(nullptr, svc::ResultInvalidState()); thread->EndWait(svc::ResultInvalidState());
thread->Wakeup();
} }
} }
} else { } else {
/* If the address wasn't accessible, note so. */ /* If the address wasn't accessible, note so. */
thread->SetSyncedObject(nullptr, svc::ResultInvalidCurrentMemory()); thread->EndWait(svc::ResultInvalidCurrentMemory());
thread->Wakeup();
} }
} }
@ -190,13 +212,11 @@ namespace ams::kern {
/* Prepare to wait. */ /* Prepare to wait. */
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKConditionVariableWaitConditionVariable wait_queue(std::addressof(m_tree));
{ {
KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
/* Set the synced object. */
cur_thread->SetSyncedObject(nullptr, ams::svc::ResultTimedOut());
/* Check that the thread isn't terminating. */ /* Check that the thread isn't terminating. */
if (cur_thread->IsTerminationRequested()) { if (cur_thread->IsTerminationRequested()) {
slp.CancelSleep(); slp.CancelSleep();
@ -219,8 +239,7 @@ namespace ams::kern {
} }
/* Wake up the next owner. */ /* Wake up the next owner. */
next_owner_thread->SetSyncedObject(nullptr, ResultSuccess()); next_owner_thread->EndWait(ResultSuccess());
next_owner_thread->Wakeup();
} }
/* Write to the cv key. */ /* Write to the cv key. */
@ -237,40 +256,20 @@ namespace ams::kern {
} }
} }
/* If timeout is zero, time out. */
R_UNLESS(timeout != 0, svc::ResultTimedOut());
/* Update condition variable tracking. */ /* Update condition variable tracking. */
{
cur_thread->SetConditionVariable(std::addressof(m_tree), addr, key, value); cur_thread->SetConditionVariable(std::addressof(m_tree), addr, key, value);
m_tree.insert(*cur_thread); m_tree.insert(*cur_thread);
/* Begin waiting. */
wait_queue.SetHardwareTimer(timer);
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* If the timeout is non-zero, set the thread as waiting. */ /* Get the wait result. */
if (timeout != 0) { return cur_thread->GetWaitResult();
cur_thread->SetState(KThread::ThreadState_Waiting);
}
}
/* Remove from the condition variable. */
{
KScopedSchedulerLock sl;
if (KThread *owner = cur_thread->GetLockOwner(); owner != nullptr) {
owner->RemoveWaiter(cur_thread);
}
if (cur_thread->IsWaitingForConditionVariable()) {
m_tree.erase(m_tree.iterator_to(*cur_thread));
cur_thread->ClearConditionVariable();
}
}
/* Cancel the timer wait. */
if (timer != nullptr) {
timer->CancelTask(cur_thread);
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
} }

View file

@ -37,17 +37,7 @@ namespace ams::kern {
cur_thread->SetLightSessionData(data); cur_thread->SetLightSessionData(data);
/* Send the request. */ /* Send the request. */
{ return m_parent->OnRequest(cur_thread);
KScopedSchedulerLock sl;
cur_thread->SetSyncedObject(nullptr, ResultSuccess());
R_TRY(m_parent->OnRequest(cur_thread));
}
/* Get the result. */
KSynchronizationObject *dummy;
return cur_thread->GetWaitResult(std::addressof(dummy));
} }
} }

View file

@ -0,0 +1,84 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern {
namespace {
class ThreadQueueImplForKLightConditionVariable final : public KThreadQueue {
private:
KThread::WaiterList *m_wait_list;
bool m_allow_terminating_thread;
public:
constexpr ThreadQueueImplForKLightConditionVariable(KThread::WaiterList *wl, bool term) : KThreadQueue(), m_wait_list(wl), m_allow_terminating_thread(term) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Only process waits if we're allowed to. */
if (svc::ResultTerminationRequested::Includes(wait_result) && m_allow_terminating_thread) {
return;
}
/* Remove the thread from the waiting thread from the light condition variable. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KLightConditionVariable::Wait(KLightLock *lock, s64 timeout, bool allow_terminating_thread) {
/* Create thread queue. */
KThread *owner = GetCurrentThreadPointer();
KHardwareTimer *timer;
ThreadQueueImplForKLightConditionVariable wait_queue(std::addressof(m_wait_list), allow_terminating_thread);
/* Sleep the thread. */
{
KScopedSchedulerLockAndSleep lk(&timer, owner, timeout);
if (!allow_terminating_thread && owner->IsTerminationRequested()) {
lk.CancelSleep();
return;
}
lock->Unlock();
/* Add the thread to the queue. */
m_wait_list.push_back(*owner);
/* Begin waiting. */
wait_queue.SetHardwareTimer(timer);
owner->BeginWait(std::addressof(wait_queue));
}
/* Re-acquire the lock. */
lock->Lock();
}
void KLightConditionVariable::Broadcast() {
KScopedSchedulerLock lk;
/* Signal all threads. */
for (auto it = m_wait_list.begin(); it != m_wait_list.end(); it = m_wait_list.erase(it)) {
it->EndWait(ResultSuccess());
}
}
}

View file

@ -17,8 +17,23 @@
namespace ams::kern { namespace ams::kern {
namespace {
class ThreadQueueImplForKLightLock final : public KThreadQueue {
public:
constexpr ThreadQueueImplForKLightLock() : KThreadQueue() { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Do nothing, waiting to acquire a light lock cannot be canceled. */
MESOSPHERE_UNUSED(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KLightLock::LockSlowPath(uintptr_t _owner, uintptr_t _cur_thread) { void KLightLock::LockSlowPath(uintptr_t _owner, uintptr_t _cur_thread) {
KThread *cur_thread = reinterpret_cast<KThread *>(_cur_thread); KThread *cur_thread = reinterpret_cast<KThread *>(_cur_thread);
ThreadQueueImplForKLightLock wait_queue;
/* Pend the current thread waiting on the owner thread. */ /* Pend the current thread waiting on the owner thread. */
{ {
@ -34,22 +49,13 @@ namespace ams::kern {
cur_thread->SetAddressKey(reinterpret_cast<uintptr_t>(std::addressof(m_tag))); cur_thread->SetAddressKey(reinterpret_cast<uintptr_t>(std::addressof(m_tag)));
owner_thread->AddWaiter(cur_thread); owner_thread->AddWaiter(cur_thread);
/* Set thread states. */ /* Begin waiting to hold the lock. */
cur_thread->SetState(KThread::ThreadState_Waiting); cur_thread->BeginWait(std::addressof(wait_queue));
if (owner_thread->IsSuspended()) { if (owner_thread->IsSuspended()) {
owner_thread->ContinueIfHasKernelWaiters(); owner_thread->ContinueIfHasKernelWaiters();
} }
} }
/* We're no longer waiting on the lock owner. */
{
KScopedSchedulerLock sl;
if (KThread *owner_thread = cur_thread->GetLockOwner(); AMS_UNLIKELY(owner_thread != nullptr)) {
owner_thread->RemoveWaiter(cur_thread);
}
}
} }
void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) { void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) {
@ -71,7 +77,7 @@ namespace ams::kern {
next_tag |= 0x1; next_tag |= 0x1;
} }
next_owner->SetState(KThread::ThreadState_Runnable); next_owner->EndWait(ResultSuccess());
if (next_owner->IsSuspended()) { if (next_owner->IsSuspended()) {
next_owner->ContinueIfHasKernelWaiters(); next_owner->ContinueIfHasKernelWaiters();

View file

@ -17,6 +17,64 @@
namespace ams::kern { namespace ams::kern {
namespace {
constexpr u64 InvalidThreadId = -1ull;
class ThreadQueueImplForKLightServerSessionRequest final : public KThreadQueue {
private:
KThread::WaiterList *m_wait_list;
public:
constexpr ThreadQueueImplForKLightServerSessionRequest(KThread::WaiterList *wl) : KThreadQueue(), m_wait_list(wl) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Remove the thread from our wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread from our wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
class ThreadQueueImplForKLightServerSessionReceive final : public KThreadQueue {
private:
KThread **m_server_thread;
public:
constexpr ThreadQueueImplForKLightServerSessionReceive(KThread **st) : KThreadQueue(), m_server_thread(st) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Clear the server thread. */
*m_server_thread = nullptr;
/* Set the waiting thread as not cancelable. */
waiting_thread->ClearCancellable();
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Clear the server thread. */
*m_server_thread = nullptr;
/* Set the waiting thread as not cancelable. */
waiting_thread->ClearCancellable();
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KLightServerSession::Destroy() { void KLightServerSession::Destroy() {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
@ -33,31 +91,45 @@ namespace ams::kern {
Result KLightServerSession::OnRequest(KThread *request_thread) { Result KLightServerSession::OnRequest(KThread *request_thread) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
ThreadQueueImplForKLightServerSessionRequest wait_queue(std::addressof(m_request_list));
/* Send the request. */
{
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Check that the server isn't closed. */ /* Check that the server isn't closed. */
R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed());
/* Try to sleep the thread. */ /* Check that the request thread isn't terminating. */
R_UNLESS(m_request_queue.SleepThread(request_thread), svc::ResultTerminationRequested()); R_UNLESS(!request_thread->IsTerminationRequested(), svc::ResultTerminationRequested());
/* If we don't have a current request, wake up a server thread to handle it. */ /* Add the request thread to our list. */
if (m_current_request == nullptr) { m_request_list.push_back(*request_thread);
m_server_queue.WakeupFrontThread();
/* Begin waiting on the request. */
request_thread->BeginWait(std::addressof(wait_queue));
/* If we have a server thread, end its wait. */
if (m_server_thread != nullptr) {
m_server_thread->EndWait(ResultSuccess());
}
} }
return ResultSuccess(); /* NOTE: Nintendo returns GetCurrentThread().GetWaitResult() here. */
/* This is technically incorrect, although it doesn't cause problems in practice */
/* because this is only ever called with request_thread = GetCurrentThreadPointer(). */
return request_thread->GetWaitResult();
} }
Result KLightServerSession::ReplyAndReceive(u32 *data) { Result KLightServerSession::ReplyAndReceive(u32 *data) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
/* Set the server context. */ /* Set the server context. */
KThread *server_thread = GetCurrentThreadPointer(); GetCurrentThread().SetLightSessionData(data);
server_thread->SetLightSessionData(data);
/* Reply, if we need to. */ /* Reply, if we need to. */
KThread *cur_request = nullptr;
if (data[0] & KLightSession::ReplyFlag) { if (data[0] & KLightSession::ReplyFlag) {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
@ -68,78 +140,85 @@ namespace ams::kern {
/* Check that we have a request to reply to. */ /* Check that we have a request to reply to. */
R_UNLESS(m_current_request != nullptr, svc::ResultInvalidState()); R_UNLESS(m_current_request != nullptr, svc::ResultInvalidState());
/* Check that the server thread is correct. */ /* Check that the server thread id is correct. */
R_UNLESS(m_server_thread == server_thread, svc::ResultInvalidState()); R_UNLESS(m_server_thread_id == GetCurrentThread().GetId(), svc::ResultInvalidState());
/* If we can reply, do so. */ /* If we can reply, do so. */
if (!m_current_request->IsTerminationRequested()) { if (!m_current_request->IsTerminationRequested()) {
MESOSPHERE_ASSERT(m_current_request->GetState() == KThread::ThreadState_Waiting); std::memcpy(m_current_request->GetLightSessionData(), GetCurrentThread().GetLightSessionData(), KLightSession::DataSize);
MESOSPHERE_ASSERT(m_request_queue.begin() != m_request_queue.end() && m_current_request == std::addressof(*m_request_queue.begin())); m_current_request->EndWait(ResultSuccess());
std::memcpy(m_current_request->GetLightSessionData(), server_thread->GetLightSessionData(), KLightSession::DataSize);
m_request_queue.WakeupThread(m_current_request);
} }
/* Close our current request. */
m_current_request->Close();
/* Clear our current request. */ /* Clear our current request. */
cur_request = m_current_request;
m_current_request = nullptr; m_current_request = nullptr;
m_server_thread = nullptr; m_server_thread_id = InvalidThreadId;
} }
/* Close the current request, if we had one. */ /* Close any pending objects before we wait. */
if (cur_request != nullptr) { GetCurrentThread().DestroyClosedObjects();
cur_request->Close();
} /* Create the wait queue for our receive. */
ThreadQueueImplForKLightServerSessionReceive wait_queue(std::addressof(m_server_thread));
/* Receive. */ /* Receive. */
bool set_cancellable = false;
while (true) { while (true) {
/* Try to receive a request. */
{
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
/* Check that we aren't already receiving. */ /* Check that we aren't already receiving. */
R_UNLESS(m_server_queue.IsEmpty(), svc::ResultInvalidState());
R_UNLESS(m_server_thread == nullptr, svc::ResultInvalidState()); R_UNLESS(m_server_thread == nullptr, svc::ResultInvalidState());
R_UNLESS(m_server_thread_id == InvalidThreadId, svc::ResultInvalidState());
/* If we cancelled in a previous loop, clear cancel state. */
if (set_cancellable) {
server_thread->ClearCancellable();
set_cancellable = false;
}
/* Check that we're open. */ /* Check that we're open. */
R_UNLESS(!m_parent->IsClientClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsClientClosed(), svc::ResultSessionClosed());
R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed());
/* Check that we're not terminating. */
R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
/* If we have a request available, use it. */ /* If we have a request available, use it. */
if (m_current_request == nullptr && !m_request_queue.IsEmpty()) { if (auto head = m_request_list.begin(); head != m_request_list.end()) {
m_current_request = std::addressof(*m_request_queue.begin()); /* Set our current request. */
m_current_request = std::addressof(*head);
m_current_request->Open(); m_current_request->Open();
m_server_thread = server_thread;
break; /* Set our server thread id. */
} else { m_server_thread_id = GetCurrentThread().GetId();
/* Otherwise, wait for a request to come in. */
R_UNLESS(m_server_queue.SleepThread(server_thread), svc::ResultTerminationRequested()); /* Copy the client request data. */
std::memcpy(GetCurrentThread().GetLightSessionData(), m_current_request->GetLightSessionData(), KLightSession::DataSize);
/* We successfully received. */
return ResultSuccess();
}
/* We need to wait for a request to come in. */
/* Check if we were cancelled. */ /* Check if we were cancelled. */
if (server_thread->IsWaitCancelled()) { if (GetCurrentThread().IsWaitCancelled()) {
m_server_queue.WakeupThread(server_thread); GetCurrentThread().ClearWaitCancelled();
server_thread->ClearWaitCancelled();
return svc::ResultCancelled(); return svc::ResultCancelled();
} }
/* Otherwise, mark as cancellable. */ /* Mark ourselves as cancellable. */
server_thread->SetCancellable(); GetCurrentThread().SetCancellable();
set_cancellable = true;
} /* Wait for a request to come in. */
m_server_thread = GetCurrentThreadPointer();
GetCurrentThread().BeginWait(std::addressof(wait_queue));
} }
/* Copy the client data. */ /* We waited to receive a request; if our wait failed, return the failing result. */
std::memcpy(server_thread->GetLightSessionData(), m_current_request->GetLightSessionData(), KLightSession::DataSize); R_TRY(GetCurrentThread().GetWaitResult());
return ResultSuccess(); }
} }
void KLightServerSession::CleanupRequests() { void KLightServerSession::CleanupRequests() {
/* Cleanup all pending requests. */ /* Cleanup all pending requests. */
KThread *cur_request = nullptr;
{ {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
@ -147,34 +226,25 @@ namespace ams::kern {
if (m_current_request != nullptr) { if (m_current_request != nullptr) {
/* Reply to the current request. */ /* Reply to the current request. */
if (!m_current_request->IsTerminationRequested()) { if (!m_current_request->IsTerminationRequested()) {
MESOSPHERE_ASSERT(m_current_request->GetState() == KThread::ThreadState_Waiting); m_current_request->EndWait(svc::ResultSessionClosed());
MESOSPHERE_ASSERT(m_request_queue.begin() != m_request_queue.end() && m_current_request == std::addressof(*m_request_queue.begin()));
m_request_queue.WakeupThread(m_current_request);
m_current_request->SetSyncedObject(nullptr, svc::ResultSessionClosed());
} }
/* Clear our current request. */ /* Clear our current request. */
cur_request = m_current_request; m_current_request->Close();
m_current_request = nullptr; m_current_request = nullptr;
m_server_thread = nullptr; m_server_thread_id = InvalidThreadId;
} }
/* Reply to all other requests. */ /* Reply to all other requests. */
while (!m_request_queue.IsEmpty()) { for (auto &thread : m_request_list) {
KThread *client_thread = m_request_queue.WakeupFrontThread(); thread.EndWait(svc::ResultSessionClosed());
client_thread->SetSyncedObject(nullptr, svc::ResultSessionClosed());
} }
/* Wake up all server threads. */ /* Wait up our server thread, if we have one. */
while (!m_server_queue.IsEmpty()) { if (m_server_thread != nullptr) {
m_server_queue.WakeupFrontThread(); m_server_thread->EndWait(svc::ResultSessionClosed());
} }
} }
/* Close the current request, if we had one. */
if (cur_request != nullptr) {
cur_request->Close();
}
} }
} }

View file

@ -87,6 +87,29 @@ namespace ams::kern {
return ResultSuccess(); return ResultSuccess();
} }
class ThreadQueueImplForKProcessEnterUserException final : public KThreadQueue {
private:
KThread **m_exception_thread;
public:
constexpr ThreadQueueImplForKProcessEnterUserException(KThread **t) : KThreadQueue(), m_exception_thread(t) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Set the exception thread. */
*m_exception_thread = waiting_thread;
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread as a waiter on its mutex owner. */
waiting_thread->GetLockOwner()->RemoveWaiter(waiting_thread);
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
void KProcess::Finalize() { void KProcess::Finalize() {
@ -784,43 +807,43 @@ namespace ams::kern {
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
MESOSPHERE_ASSERT(this == cur_thread->GetOwnerProcess()); MESOSPHERE_ASSERT(this == cur_thread->GetOwnerProcess());
/* Try to claim the exception thread. */ /* Check that we haven't already claimed the exception thread. */
if (m_exception_thread != cur_thread) { if (m_exception_thread == cur_thread) {
const uintptr_t address_key = reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)); return false;
while (true) { }
/* Create the wait queue we'll be using. */
ThreadQueueImplForKProcessEnterUserException wait_queue(std::addressof(m_exception_thread));
/* Claim the exception thread. */
{ {
/* Lock the scheduler. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
/* If the thread is terminating, it can't enter. */ /* Check that we're not terminating. */
if (cur_thread->IsTerminationRequested()) { if (cur_thread->IsTerminationRequested()) {
return false; return false;
} }
/* If we have no exception thread, we succeeded. */ /* If we don't have an exception thread, we can just claim it directly. */
if (m_exception_thread == nullptr) { if (m_exception_thread == nullptr) {
m_exception_thread = cur_thread; m_exception_thread = cur_thread;
KScheduler::SetSchedulerUpdateNeeded(); KScheduler::SetSchedulerUpdateNeeded();
return true; return true;
} }
/* Otherwise, wait for us to not have an exception thread. */ /* Otherwise, we need to wait until we don't have an exception thread. */
cur_thread->SetAddressKey(address_key | 1);
/* Add the current thread as a waiter on the current exception thread. */
cur_thread->SetAddressKey(reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)) | 1);
m_exception_thread->AddWaiter(cur_thread); m_exception_thread->AddWaiter(cur_thread);
cur_thread->SetState(KThread::ThreadState_Waiting);
/* Wait to claim the exception thread. */
cur_thread->BeginWait(std::addressof(wait_queue));
} }
/* Remove the thread as a waiter from the lock owner. */ /* If our wait didn't end due to thread termination, we succeeded. */
{ return !svc::ResultTerminationRequested::Includes(cur_thread->GetWaitResult());
KScopedSchedulerLock sl;
if (KThread *owner_thread = cur_thread->GetLockOwner(); owner_thread != nullptr) {
owner_thread->RemoveWaiter(cur_thread);
}
}
}
} else {
return false;
}
} }
bool KProcess::LeaveUserException() { bool KProcess::LeaveUserException() {
@ -836,7 +859,7 @@ namespace ams::kern {
/* Remove waiter thread. */ /* Remove waiter thread. */
s32 num_waiters; s32 num_waiters;
if (KThread *next = thread->RemoveWaiterByKey(std::addressof(num_waiters), reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)) | 1); next != nullptr) { if (KThread *next = thread->RemoveWaiterByKey(std::addressof(num_waiters), reinterpret_cast<uintptr_t>(std::addressof(m_exception_thread)) | 1); next != nullptr) {
next->SetState(KThread::ThreadState_Runnable); next->EndWait(ResultSuccess());
} }
KScheduler::SetSchedulerUpdateNeeded(); KScheduler::SetSchedulerUpdateNeeded();

View file

@ -30,6 +30,8 @@ namespace ams::kern {
constexpr inline size_t PointerTransferBufferAlignment = 0x10; constexpr inline size_t PointerTransferBufferAlignment = 0x10;
class ThreadQueueImplForKServerSessionRequest final : public KThreadQueue { /* ... */ };
class ReceiveList { class ReceiveList {
private: private:
u32 m_data[ipc::MessageBuffer::MessageHeader::ReceiveListCountType_CountMax * ipc::MessageBuffer::ReceiveListEntry::GetDataSize() / sizeof(u32)]; u32 m_data[ipc::MessageBuffer::MessageHeader::ReceiveListCountType_CountMax * ipc::MessageBuffer::ReceiveListEntry::GetDataSize() / sizeof(u32)];
@ -1042,11 +1044,11 @@ namespace ams::kern {
/* Signal the event. */ /* Signal the event. */
event->Signal(); event->Signal();
} else { } else {
/* Set the thread as runnable. */ /* End the client thread's wait. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
if (client_thread->GetState() == KThread::ThreadState_Waiting) {
client_thread->SetSyncedObject(nullptr, result_for_client); if (!client_thread->IsTerminationRequested()) {
client_thread->SetState(KThread::ThreadState_Runnable); client_thread->EndWait(result_for_client);
} }
} }
} }
@ -1146,11 +1148,11 @@ namespace ams::kern {
/* Signal the event. */ /* Signal the event. */
event->Signal(); event->Signal();
} else { } else {
/* Set the thread as runnable. */ /* End the client thread's wait. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
if (client_thread->GetState() == KThread::ThreadState_Waiting) {
client_thread->SetSyncedObject(nullptr, client_result); if (!client_thread->IsTerminationRequested()) {
client_thread->SetState(KThread::ThreadState_Runnable); client_thread->EndWait(client_result);
} }
} }
} }
@ -1160,17 +1162,20 @@ namespace ams::kern {
Result KServerSession::OnRequest(KSessionRequest *request) { Result KServerSession::OnRequest(KSessionRequest *request) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
/* Create the wait queue. */
ThreadQueueImplForKServerSessionRequest wait_queue;
/* Handle the request. */
{
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Ensure that we can handle new requests. */ /* Ensure that we can handle new requests. */
R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed()); R_UNLESS(!m_parent->IsServerClosed(), svc::ResultSessionClosed());
/* If there's no event, this is synchronous, so we should check for thread termination. */ /* Check that we're not terminating. */
if (request->GetEvent() == nullptr) { R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
KThread *thread = request->GetThread();
R_UNLESS(!thread->IsTerminationRequested(), svc::ResultTerminationRequested());
thread->SetState(KThread::ThreadState_Waiting);
}
/* Get whether we're empty. */ /* Get whether we're empty. */
const bool was_empty = m_request_list.empty(); const bool was_empty = m_request_list.empty();
@ -1184,7 +1189,14 @@ namespace ams::kern {
this->NotifyAvailable(); this->NotifyAvailable();
} }
return ResultSuccess(); /* If we have a request, this is asynchronous, and we don't need to wait. */
R_SUCCEED_IF(request->GetEvent() != nullptr);
/* This is a synchronous request, so we should wait for our request to complete. */
GetCurrentThread().BeginWait(std::addressof(wait_queue));
}
return GetCurrentThread().GetWaitResult();
} }
bool KServerSession::IsSignaledImpl() const { bool KServerSession::IsSignaledImpl() const {
@ -1264,11 +1276,11 @@ namespace ams::kern {
/* Signal the event. */ /* Signal the event. */
event->Signal(); event->Signal();
} else { } else {
/* Set the thread as runnable. */ /* End the client thread's wait. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
if (client_thread->GetState() == KThread::ThreadState_Waiting) {
client_thread->SetSyncedObject(nullptr, (R_SUCCEEDED(result) ? svc::ResultSessionClosed() : result)); if (!client_thread->IsTerminationRequested()) {
client_thread->SetState(KThread::ThreadState_Runnable); client_thread->EndWait(R_SUCCEEDED(result) ? svc::ResultSessionClosed() : result);
} }
} }
} }
@ -1310,6 +1322,7 @@ namespace ams::kern {
request->ClearEvent(); request->ClearEvent();
terminate = true; terminate = true;
} }
prev_request = request; prev_request = request;
} else if (!m_request_list.empty()) { } else if (!m_request_list.empty()) {
/* Pop the request from the front of the list. */ /* Pop the request from the front of the list. */

View file

@ -17,6 +17,57 @@
namespace ams::kern { namespace ams::kern {
namespace {
class ThreadQueueImplForKSynchronizationObjectWait final : public KThreadQueueWithoutEndWait {
private:
using ThreadListNode = KSynchronizationObject::ThreadListNode;
private:
KSynchronizationObject **m_objects;
ThreadListNode *m_nodes;
s32 m_count;
public:
constexpr ThreadQueueImplForKSynchronizationObjectWait(KSynchronizationObject **o, ThreadListNode *n, s32 c) : m_objects(o), m_nodes(n), m_count(c) { /* ... */ }
virtual void NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result) override {
/* Determine the sync index, and unlink all nodes. */
s32 sync_index = -1;
for (auto i = 0; i < m_count; ++i) {
/* Check if this is the signaled object. */
if (m_objects[i] == signaled_object && sync_index == -1) {
sync_index = i;
}
/* Unlink the current node from the current object. */
m_objects[i]->UnlinkNode(std::addressof(m_nodes[i]));
}
/* Set the waiting thread's sync index. */
waiting_thread->SetSyncedIndex(sync_index);
/* Set the waiting thread as not cancellable. */
waiting_thread->ClearCancellable();
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove all nodes from our list. */
for (auto i = 0; i < m_count; ++i) {
m_objects[i]->UnlinkNode(std::addressof(m_nodes[i]));
}
/* Set the waiting thread as not cancellable. */
waiting_thread->ClearCancellable();
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
void KSynchronizationObject::Finalize() { void KSynchronizationObject::Finalize() {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
@ -43,14 +94,21 @@ namespace ams::kern {
/* Prepare for wait. */ /* Prepare for wait. */
KThread *thread = GetCurrentThreadPointer(); KThread *thread = GetCurrentThreadPointer();
KHardwareTimer *timer; KHardwareTimer *timer;
ThreadQueueImplForKSynchronizationObjectWait wait_queue(objects, thread_nodes, num_objects);
{ {
/* Setup the scheduling lock and sleep. */ /* Setup the scheduling lock and sleep. */
KScopedSchedulerLockAndSleep slp(std::addressof(timer), thread, timeout); KScopedSchedulerLockAndSleep slp(std::addressof(timer), thread, timeout);
/* Check if the thread should terminate. */
if (thread->IsTerminationRequested()) {
slp.CancelSleep();
return svc::ResultTerminationRequested();
}
/* Check if any of the objects are already signaled. */ /* Check if any of the objects are already signaled. */
for (auto i = 0; i < num_objects; ++i) { for (auto i = 0; i < num_objects; ++i) {
AMS_ASSERT(objects[i] != nullptr); MESOSPHERE_ASSERT(objects[i] != nullptr);
if (objects[i]->IsSignaled()) { if (objects[i]->IsSignaled()) {
*out_index = i; *out_index = i;
@ -65,12 +123,6 @@ namespace ams::kern {
return svc::ResultTimedOut(); return svc::ResultTimedOut();
} }
/* Check if the thread should terminate. */
if (thread->IsTerminationRequested()) {
slp.CancelSleep();
return svc::ResultTerminationRequested();
}
/* Check if waiting was canceled. */ /* Check if waiting was canceled. */
if (thread->IsWaitCancelled()) { if (thread->IsWaitCancelled()) {
slp.CancelSleep(); slp.CancelSleep();
@ -83,67 +135,25 @@ namespace ams::kern {
thread_nodes[i].thread = thread; thread_nodes[i].thread = thread;
thread_nodes[i].next = nullptr; thread_nodes[i].next = nullptr;
if (objects[i]->m_thread_list_tail == nullptr) { objects[i]->LinkNode(std::addressof(thread_nodes[i]));
objects[i]->m_thread_list_head = std::addressof(thread_nodes[i]);
} else {
objects[i]->m_thread_list_tail->next = std::addressof(thread_nodes[i]);
} }
objects[i]->m_thread_list_tail = std::addressof(thread_nodes[i]); /* Mark the thread as cancellable. */
}
/* Mark the thread as waiting. */
thread->SetCancellable(); thread->SetCancellable();
thread->SetSyncedObject(nullptr, svc::ResultTimedOut());
thread->SetState(KThread::ThreadState_Waiting); /* Clear the thread's synced index. */
thread->SetSyncedIndex(-1);
/* Wait for an object to be signaled. */
wait_queue.SetHardwareTimer(timer);
thread->BeginWait(std::addressof(wait_queue));
} }
/* The lock/sleep is done, so we should be able to get our result. */ /* Set the output index. */
*out_index = thread->GetSyncedIndex();
/* Thread is no longer cancellable. */
thread->ClearCancellable();
/* Cancel the timer as needed. */
if (timer != nullptr) {
timer->CancelTask(thread);
}
/* Get the wait result. */ /* Get the wait result. */
Result wait_result; return thread->GetWaitResult();
s32 sync_index = -1;
{
KScopedSchedulerLock lk;
KSynchronizationObject *synced_obj;
wait_result = thread->GetWaitResult(std::addressof(synced_obj));
for (auto i = 0; i < num_objects; ++i) {
/* Unlink the object from the list. */
ThreadListNode *prev_ptr = reinterpret_cast<ThreadListNode *>(std::addressof(objects[i]->m_thread_list_head));
ThreadListNode *prev_val = nullptr;
ThreadListNode *prev, *tail_prev;
do {
prev = prev_ptr;
prev_ptr = prev_ptr->next;
tail_prev = prev_val;
prev_val = prev_ptr;
} while (prev_ptr != std::addressof(thread_nodes[i]));
if (objects[i]->m_thread_list_tail == std::addressof(thread_nodes[i])) {
objects[i]->m_thread_list_tail = tail_prev;
}
prev->next = thread_nodes[i].next;
if (objects[i] == synced_obj) {
sync_index = i;
}
}
}
/* Set output. */
*out_index = sync_index;
return wait_result;
} }
void KSynchronizationObject::NotifyAvailable(Result result) { void KSynchronizationObject::NotifyAvailable(Result result) {
@ -158,11 +168,7 @@ namespace ams::kern {
/* Iterate over each thread. */ /* Iterate over each thread. */
for (auto *cur_node = m_thread_list_head; cur_node != nullptr; cur_node = cur_node->next) { for (auto *cur_node = m_thread_list_head; cur_node != nullptr; cur_node = cur_node->next) {
KThread *thread = cur_node->thread; cur_node->thread->NotifyAvailable(this, result);
if (thread->GetState() == KThread::ThreadState_Waiting) {
thread->SetSyncedObject(this, result);
thread->SetState(KThread::ThreadState_Runnable);
}
} }
} }

View file

@ -47,6 +47,23 @@ namespace ams::kern {
KPageBuffer::Free(KPageBuffer::FromPhysicalAddress(stack_paddr)); KPageBuffer::Free(KPageBuffer::FromPhysicalAddress(stack_paddr));
} }
class ThreadQueueImplForKThreadSleep final : public KThreadQueueWithoutEndWait { /* ... */ };
class ThreadQueueImplForKThreadSetProperty final : public KThreadQueue {
private:
KThread::WaiterList *m_wait_list;
public:
constexpr ThreadQueueImplForKThreadSetProperty(KThread::WaiterList *wl) : m_wait_list(wl) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread from the wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
} }
Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type) { Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type) {
@ -131,12 +148,12 @@ namespace ams::kern {
m_priority = prio; m_priority = prio;
m_base_priority = prio; m_base_priority = prio;
/* Set sync object and waiting lock to null. */ /* Set waiting lock to null. */
m_synced_object = nullptr;
m_waiting_lock = nullptr; m_waiting_lock = nullptr;
/* Initialize sleeping queue. */ /* Initialize wait queue/sync index. */
m_sleeping_queue = nullptr; m_synced_index = -1;
m_wait_queue = nullptr;
/* Set suspend flags. */ /* Set suspend flags. */
m_suspend_request_flags = 0; m_suspend_request_flags = 0;
@ -295,12 +312,20 @@ namespace ams::kern {
auto it = m_waiter_list.begin(); auto it = m_waiter_list.begin();
while (it != m_waiter_list.end()) { while (it != m_waiter_list.end()) {
/* Get the thread. */
KThread * const waiter = std::addressof(*it);
/* The thread shouldn't be a kernel waiter. */ /* The thread shouldn't be a kernel waiter. */
MESOSPHERE_ASSERT(!IsKernelAddressKey(it->GetAddressKey())); MESOSPHERE_ASSERT(!IsKernelAddressKey(waiter->GetAddressKey()));
it->SetLockOwner(nullptr);
it->SetSyncedObject(nullptr, svc::ResultInvalidState()); /* Clear the lock owner. */
it->Wakeup(); waiter->SetLockOwner(nullptr);
/* Erase the waiter from our list. */
it = m_waiter_list.erase(it); it = m_waiter_list.erase(it);
/* Cancel the thread's wait. */
waiter->CancelWait(svc::ResultInvalidState(), true);
} }
} }
@ -320,24 +345,14 @@ namespace ams::kern {
return m_signaled; return m_signaled;
} }
void KThread::Wakeup() {
MESOSPHERE_ASSERT_THIS();
KScopedSchedulerLock sl;
if (this->GetState() == ThreadState_Waiting) {
if (m_sleeping_queue != nullptr) {
m_sleeping_queue->WakeupThread(this);
} else {
this->SetState(ThreadState_Runnable);
}
}
}
void KThread::OnTimer() { void KThread::OnTimer() {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
this->Wakeup(); /* If we're waiting, cancel the wait. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->CancelWait(this, svc::ResultTimedOut(), false);
}
} }
void KThread::StartTermination() { void KThread::StartTermination() {
@ -362,7 +377,7 @@ namespace ams::kern {
/* Signal. */ /* Signal. */
m_signaled = true; m_signaled = true;
this->NotifyAvailable(); KSynchronizationObject::NotifyAvailable();
/* Call the on thread termination handler. */ /* Call the on thread termination handler. */
KThreadContext::OnThreadTerminating(this); KThreadContext::OnThreadTerminating(this);
@ -496,10 +511,8 @@ namespace ams::kern {
} }
/* Resume any threads that began waiting on us while we were pinned. */ /* Resume any threads that began waiting on us while we were pinned. */
for (auto it = m_pinned_waiter_list.begin(); it != m_pinned_waiter_list.end(); ++it) { for (auto it = m_pinned_waiter_list.begin(); it != m_pinned_waiter_list.end(); it = m_pinned_waiter_list.erase(it)) {
if (it->GetState() == ThreadState_Waiting) { it->EndWait(ResultSuccess());
it->SetState(ThreadState_Runnable);
}
} }
} }
@ -646,9 +659,9 @@ namespace ams::kern {
} }
/* Update the pinned waiter list. */ /* Update the pinned waiter list. */
ThreadQueueImplForKThreadSetProperty wait_queue(std::addressof(m_pinned_waiter_list));
{ {
bool retry_update; bool retry_update;
bool thread_is_pinned = false;
do { do {
/* Lock the scheduler. */ /* Lock the scheduler. */
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
@ -676,27 +689,15 @@ namespace ams::kern {
/* Verify that the current thread isn't terminating. */ /* Verify that the current thread isn't terminating. */
R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested()); R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
/* Note that the thread was pinned. */
thread_is_pinned = true;
/* Wait until the thread isn't pinned any more. */ /* Wait until the thread isn't pinned any more. */
m_pinned_waiter_list.push_back(GetCurrentThread()); m_pinned_waiter_list.push_back(GetCurrentThread());
GetCurrentThread().SetState(ThreadState_Waiting); GetCurrentThread().BeginWait(std::addressof(wait_queue));
} else { } else {
/* If the thread isn't pinned, release the scheduler lock and retry until it's not current. */ /* If the thread isn't pinned, release the scheduler lock and retry until it's not current. */
retry_update = true; retry_update = true;
} }
} }
} while (retry_update); } while (retry_update);
/* If the thread was pinned, it no longer is, and we should remove the current thread from our waiter list. */
if (thread_is_pinned) {
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Remove from the list. */
m_pinned_waiter_list.erase(m_pinned_waiter_list.iterator_to(GetCurrentThread()));
}
} }
return ResultSuccess(); return ResultSuccess();
@ -785,14 +786,8 @@ namespace ams::kern {
/* Check if we're waiting and cancellable. */ /* Check if we're waiting and cancellable. */
if (this->GetState() == ThreadState_Waiting && m_cancellable) { if (this->GetState() == ThreadState_Waiting && m_cancellable) {
if (m_sleeping_queue != nullptr) {
m_sleeping_queue->WakeupThread(this);
m_wait_cancelled = true;
} else {
this->SetSyncedObject(nullptr, svc::ResultCancelled());
this->SetState(ThreadState_Runnable);
m_wait_cancelled = false; m_wait_cancelled = false;
} m_wait_queue->CancelWait(this, svc::ResultCancelled(), true);
} else { } else {
/* Otherwise, note that we cancelled a wait. */ /* Otherwise, note that we cancelled a wait. */
m_wait_cancelled = true; m_wait_cancelled = true;
@ -894,7 +889,8 @@ namespace ams::kern {
/* If the thread is now paused, update the pinned waiter list. */ /* If the thread is now paused, update the pinned waiter list. */
if (activity == ams::svc::ThreadActivity_Paused) { if (activity == ams::svc::ThreadActivity_Paused) {
bool thread_is_pinned = false; ThreadQueueImplForKThreadSetProperty wait_queue(std::addressof(m_pinned_waiter_list));
bool thread_is_current; bool thread_is_current;
do { do {
/* Lock the scheduler. */ /* Lock the scheduler. */
@ -903,23 +899,20 @@ namespace ams::kern {
/* Don't do any further management if our termination has been requested. */ /* Don't do any further management if our termination has been requested. */
R_SUCCEED_IF(this->IsTerminationRequested()); R_SUCCEED_IF(this->IsTerminationRequested());
/* By default, treat the thread as not current. */
thread_is_current = false;
/* Check whether the thread is pinned. */ /* Check whether the thread is pinned. */
if (this->GetStackParameters().is_pinned) { if (this->GetStackParameters().is_pinned) {
/* Verify that the current thread isn't terminating. */ /* Verify that the current thread isn't terminating. */
R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested()); R_UNLESS(!GetCurrentThread().IsTerminationRequested(), svc::ResultTerminationRequested());
/* Note that the thread was pinned and not current. */
thread_is_pinned = true;
thread_is_current = false;
/* Wait until the thread isn't pinned any more. */ /* Wait until the thread isn't pinned any more. */
m_pinned_waiter_list.push_back(GetCurrentThread()); m_pinned_waiter_list.push_back(GetCurrentThread());
GetCurrentThread().SetState(ThreadState_Waiting); GetCurrentThread().BeginWait(std::addressof(wait_queue));
} else { } else {
/* Check if the thread is currently running. */ /* Check if the thread is currently running. */
/* If it is, we'll need to retry. */ /* If it is, we'll need to retry. */
thread_is_current = false;
for (auto i = 0; i < static_cast<s32>(cpu::NumCores); ++i) { for (auto i = 0; i < static_cast<s32>(cpu::NumCores); ++i) {
if (Kernel::GetScheduler(i).GetSchedulerCurrentThread() == this) { if (Kernel::GetScheduler(i).GetSchedulerCurrentThread() == this) {
thread_is_current = true; thread_is_current = true;
@ -928,15 +921,6 @@ namespace ams::kern {
} }
} }
} while (thread_is_current); } while (thread_is_current);
/* If the thread was pinned, it no longer is, and we should remove the current thread from our waiter list. */
if (thread_is_pinned) {
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* Remove from the list. */
m_pinned_waiter_list.erase(m_pinned_waiter_list.iterator_to(GetCurrentThread()));
}
} }
return ResultSuccess(); return ResultSuccess();
@ -1241,8 +1225,9 @@ namespace ams::kern {
} }
/* Wake up the thread. */ /* Wake up the thread. */
this->SetSyncedObject(nullptr, svc::ResultTerminationRequested()); if (this->GetState() == ThreadState_Waiting) {
this->Wakeup(); m_wait_queue->CancelWait(this, svc::ResultTerminationRequested(), true);
}
} }
return this->GetState(); return this->GetState();
@ -1254,6 +1239,7 @@ namespace ams::kern {
MESOSPHERE_ASSERT(this == GetCurrentThreadPointer()); MESOSPHERE_ASSERT(this == GetCurrentThreadPointer());
MESOSPHERE_ASSERT(timeout > 0); MESOSPHERE_ASSERT(timeout > 0);
ThreadQueueImplForKThreadSleep wait_queue;
KHardwareTimer *timer; KHardwareTimer *timer;
{ {
/* Setup the scheduling lock and sleep. */ /* Setup the scheduling lock and sleep. */
@ -1265,18 +1251,58 @@ namespace ams::kern {
return svc::ResultTerminationRequested(); return svc::ResultTerminationRequested();
} }
/* Mark the thread as waiting. */ /* Wait for the sleep to end. */
this->SetState(KThread::ThreadState_Waiting); wait_queue.SetHardwareTimer(timer);
this->BeginWait(std::addressof(wait_queue));
} }
/* The lock/sleep is done. */
/* Cancel the timer. */
timer->CancelTask(this);
return ResultSuccess(); return ResultSuccess();
} }
void KThread::BeginWait(KThreadQueue *queue) {
/* Set our state as waiting. */
this->SetState(ThreadState_Waiting);
/* Set our wait queue. */
m_wait_queue = queue;
}
void KThread::NotifyAvailable(KSynchronizationObject *signaled_object, Result wait_result) {
MESOSPHERE_ASSERT_THIS();
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* If we're waiting, notify our queue that we're available. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->NotifyAvailable(this, signaled_object, wait_result);
}
}
void KThread::EndWait(Result wait_result) {
MESOSPHERE_ASSERT_THIS();
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* If we're waiting, notify our queue that we're available. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->EndWait(this, wait_result);
}
}
void KThread::CancelWait(Result wait_result, bool cancel_timer_task) {
MESOSPHERE_ASSERT_THIS();
/* Lock the scheduler. */
KScopedSchedulerLock sl;
/* If we're waiting, notify our queue that we're available. */
if (this->GetState() == ThreadState_Waiting) {
m_wait_queue->CancelWait(this, wait_result, cancel_timer_task);
}
}
void KThread::SetState(ThreadState state) { void KThread::SetState(ThreadState state) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();

View file

@ -0,0 +1,62 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern {
void KThreadQueue::NotifyAvailable(KThread *waiting_thread, KSynchronizationObject *signaled_object, Result wait_result) {
MESOSPHERE_UNUSED(waiting_thread, signaled_object, wait_result);
MESOSPHERE_PANIC("KThreadQueue::NotifyAvailable\n");
}
void KThreadQueue::EndWait(KThread *waiting_thread, Result wait_result) {
/* Set the thread's wait result. */
waiting_thread->SetWaitResult(wait_result);
/* Set the thread as runnable. */
waiting_thread->SetState(KThread::ThreadState_Runnable);
/* Clear the thread's wait queue. */
waiting_thread->ClearWaitQueue();
/* Cancel the thread task. */
if (m_hardware_timer != nullptr) {
m_hardware_timer->CancelTask(waiting_thread);
}
}
void KThreadQueue::CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) {
/* Set the thread's wait result. */
waiting_thread->SetWaitResult(wait_result);
/* Set the thread as runnable. */
waiting_thread->SetState(KThread::ThreadState_Runnable);
/* Clear the thread's wait queue. */
waiting_thread->ClearWaitQueue();
/* Cancel the thread task. */
if (cancel_timer_task && m_hardware_timer != nullptr) {
m_hardware_timer->CancelTask(waiting_thread);
}
}
void KThreadQueueWithoutEndWait::EndWait(KThread *waiting_thread, Result wait_result) {
MESOSPHERE_UNUSED(waiting_thread, wait_result);
MESOSPHERE_PANIC("KThreadQueueWithoutEndWait::EndWait\n");
}
}

View file

@ -17,73 +17,83 @@
namespace ams::kern { namespace ams::kern {
void KWaitObject::OnTimer() { namespace {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
/* Wake up all the waiting threads. */ class ThreadQueueImplForKWaitObjectSynchronize final : public KThreadQueueWithoutEndWait {
for (KThread &thread : m_wait_list) { private:
thread.Wakeup(); KThread::WaiterList *m_wait_list;
KThread **m_thread;
public:
constexpr ThreadQueueImplForKWaitObjectSynchronize(KThread::WaiterList *wl, KThread **t) : KThreadQueueWithoutEndWait(), m_wait_list(wl), m_thread(t) { /* ... */ }
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
/* Remove the thread from the wait list. */
m_wait_list->erase(m_wait_list->iterator_to(*waiting_thread));
/* If the result was a timeout and the thread is our wait object thread, cancel recursively. */
if (svc::ResultTimedOut::Includes(wait_result) && waiting_thread == *m_thread) {
for (auto &thread : *m_wait_list) {
thread.CancelWait(svc::ResultTimedOut(), false);
} }
} }
/* If the thread is our wait object thread, clear it. */
if (*m_thread == waiting_thread) {
*m_thread = nullptr;
}
/* Invoke the base cancel wait handler. */
KThreadQueue::CancelWait(waiting_thread, wait_result, cancel_timer_task);
}
};
}
Result KWaitObject::Synchronize(s64 timeout) { Result KWaitObject::Synchronize(s64 timeout) {
/* Perform the wait. */ /* Perform the wait. */
KHardwareTimer *timer = nullptr; KHardwareTimer *timer;
KThread *cur_thread = GetCurrentThreadPointer(); KThread *cur_thread = GetCurrentThreadPointer();
ThreadQueueImplForKWaitObjectSynchronize wait_queue(std::addressof(m_wait_list), std::addressof(m_next_thread));
{ {
KScopedSchedulerLock sl; KScopedSchedulerLockAndSleep slp(std::addressof(timer), cur_thread, timeout);
/* Check that the thread isn't terminating. */ /* Check that the thread isn't terminating. */
R_UNLESS(!cur_thread->IsTerminationRequested(), svc::ResultTerminationRequested()); if (cur_thread->IsTerminationRequested()) {
slp.CancelSleep();
/* Verify that nothing else is already waiting on the object. */ return svc::ResultTerminationRequested();
if (timeout > 0) {
R_UNLESS(!m_timer_used, svc::ResultBusy());
} }
/* Check that we're not already in use. */ /* Handle the case where timeout is non-negative/infinite. */
if (timeout >= 0) { if (timeout >= 0) {
/* Verify the timer isn't already in use. */ /* Check if we're already waiting. */
R_UNLESS(!m_timer_used, svc::ResultBusy()); if (m_next_thread != nullptr) {
} slp.CancelSleep();
return svc::ResultBusy();
/* If we need to, register our timeout. */
if (timeout > 0) {
/* Mark that we're using the timer. */
m_timer_used = true;
/* Use the timer. */
timer = std::addressof(Kernel::GetHardwareTimer());
timer->RegisterAbsoluteTask(this, timeout);
} }
/* If timeout is zero, handle the special case by canceling all waiting threads. */
if (timeout == 0) { if (timeout == 0) {
/* If we're timed out immediately, just wake up the thread. */ for (auto &thread : m_wait_list) {
this->OnTimer(); thread.CancelWait(svc::ResultTimedOut(), false);
} else { }
/* Otherwise, sleep until the timeout occurs. */
m_wait_list.push_back(GetCurrentThread()); slp.CancelSleep();
cur_thread->SetState(KThread::ThreadState_Waiting); return ResultSuccess();
cur_thread->SetSyncedObject(nullptr, svc::ResultTimedOut());
} }
} }
/* Cleanup as necessary. */ /* If the timeout isn't infinite, register it as our next timeout. */
{
KScopedSchedulerLock sl;
/* Remove from the timer. */
if (timeout > 0) { if (timeout > 0) {
MESOSPHERE_ASSERT(m_timer_used); wait_queue.SetHardwareTimer(timer);
MESOSPHERE_ASSERT(timer != nullptr); m_next_thread = cur_thread;
timer->CancelTask(this);
m_timer_used = false;
} }
/* Remove the thread from our queue. */ /* Add the current thread to our wait list. */
if (timeout != 0) { m_wait_list.push_back(*cur_thread);
m_wait_list.erase(m_wait_list.iterator_to(GetCurrentThread()));
} /* Wait until the timeout occurs. */
cur_thread->BeginWait(std::addressof(wait_queue));
} }
return ResultSuccess(); return ResultSuccess();

View file

@ -17,22 +17,46 @@
namespace ams::kern { namespace ams::kern {
namespace {
class ThreadQueueImplForKWorkerTaskManager final : public KThreadQueue {
private:
KThread **m_waiting_thread;
public:
constexpr ThreadQueueImplForKWorkerTaskManager(KThread **t) : KThreadQueue(), m_waiting_thread(t) { /* ... */ }
virtual void EndWait(KThread *waiting_thread, Result wait_result) override {
/* Clear our waiting thread. */
*m_waiting_thread = nullptr;
/* Invoke the base end wait handler. */
KThreadQueue::EndWait(waiting_thread, wait_result);
}
virtual void CancelWait(KThread *waiting_thread, Result wait_result, bool cancel_timer_task) override {
MESOSPHERE_UNUSED(waiting_thread, wait_result, cancel_timer_task);
MESOSPHERE_PANIC("ThreadQueueImplForKWorkerTaskManager::CancelWait\n");
}
};
}
void KWorkerTaskManager::Initialize(s32 priority) { void KWorkerTaskManager::Initialize(s32 priority) {
/* Reserve a thread from the system limit. */ /* Reserve a thread from the system limit. */
MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_ThreadCountMax, 1)); MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_ThreadCountMax, 1));
/* Create a new thread. */ /* Create a new thread. */
m_thread = KThread::Create(); KThread *thread = KThread::Create();
MESOSPHERE_ABORT_UNLESS(m_thread != nullptr); MESOSPHERE_ABORT_UNLESS(thread != nullptr);
/* Launch the new thread. */ /* Launch the new thread. */
MESOSPHERE_R_ABORT_UNLESS(KThread::InitializeKernelThread(m_thread, ThreadFunction, reinterpret_cast<uintptr_t>(this), priority, cpu::NumCores - 1)); MESOSPHERE_R_ABORT_UNLESS(KThread::InitializeKernelThread(thread, ThreadFunction, reinterpret_cast<uintptr_t>(this), priority, cpu::NumCores - 1));
/* Register the new thread. */ /* Register the new thread. */
KThread::Register(m_thread); KThread::Register(thread);
/* Run the thread. */ /* Run the thread. */
m_thread->Run(); thread->Run();
} }
void KWorkerTaskManager::AddTask(WorkerType type, KWorkerTask *task) { void KWorkerTaskManager::AddTask(WorkerType type, KWorkerTask *task) {
@ -45,36 +69,40 @@ namespace ams::kern {
} }
void KWorkerTaskManager::ThreadFunctionImpl() { void KWorkerTaskManager::ThreadFunctionImpl() {
/* Create wait queue. */
ThreadQueueImplForKWorkerTaskManager wait_queue(std::addressof(m_waiting_thread));
while (true) { while (true) {
KWorkerTask *task = nullptr; KWorkerTask *task;
/* Get a worker task. */ /* Get a worker task. */
{ {
KScopedSchedulerLock sl; KScopedSchedulerLock sl;
task = this->GetTask(); task = this->GetTask();
if (task == nullptr) { if (task == nullptr) {
/* If there's nothing to do, set ourselves as waiting. */ /* Wait to have a task. */
m_active = false; m_waiting_thread = GetCurrentThreadPointer();
m_thread->SetState(KThread::ThreadState_Waiting); GetCurrentThread().BeginWait(std::addressof(wait_queue));
continue; continue;
} }
m_active = true;
} }
/* Do the task. */ /* Do the task. */
task->DoWorkerTask(); task->DoWorkerTask();
/* Destroy any objects we may need to close. */ /* Destroy any objects we may need to close. */
m_thread->DestroyClosedObjects(); GetCurrentThread().DestroyClosedObjects();
} }
} }
KWorkerTask *KWorkerTaskManager::GetTask() { KWorkerTask *KWorkerTaskManager::GetTask() {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
KWorkerTask *next = m_head_task; KWorkerTask *next = m_head_task;
if (next) {
if (next != nullptr) {
/* Advance the list. */ /* Advance the list. */
if (m_head_task == m_tail_task) { if (m_head_task == m_tail_task) {
m_head_task = nullptr; m_head_task = nullptr;
@ -86,6 +114,7 @@ namespace ams::kern {
/* Clear the next task's next. */ /* Clear the next task's next. */
next->SetNextTask(nullptr); next->SetNextTask(nullptr);
} }
return next; return next;
} }
@ -102,8 +131,8 @@ namespace ams::kern {
m_tail_task = task; m_tail_task = task;
/* Make ourselves active if we need to. */ /* Make ourselves active if we need to. */
if (!m_active) { if (m_waiting_thread != nullptr) {
m_thread->SetState(KThread::ThreadState_Runnable); m_waiting_thread->EndWait(ResultSuccess());
} }
} }
} }

View file

@ -229,11 +229,14 @@ namespace ams::kern::svc {
/* Send the request. */ /* Send the request. */
MESOSPHERE_ASSERT(message != 0); MESOSPHERE_ASSERT(message != 0);
R_TRY(SendAsyncRequestWithUserBufferImpl(out_event_handle, message, buffer_size, session_handle)); const Result result = SendAsyncRequestWithUserBufferImpl(out_event_handle, message, buffer_size, session_handle);
/* We sent the request successfully. */ /* If the request succeeds (or the thread is terminating), don't unlock the user buffer. */
if (R_SUCCEEDED(result) || svc::ResultTerminationRequested::Includes(result)) {
unlock_guard.Cancel(); unlock_guard.Cancel();
return ResultSuccess(); }
return result;
} }
ALWAYS_INLINE Result ReplyAndReceive(int32_t *out_index, KUserPointer<const ams::svc::Handle *> handles, int32_t num_handles, ams::svc::Handle reply_target, int64_t timeout_ns) { ALWAYS_INLINE Result ReplyAndReceive(int32_t *out_index, KUserPointer<const ams::svc::Handle *> handles, int32_t num_handles, ams::svc::Handle reply_target, int64_t timeout_ns) {