mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2024-12-18 08:22:04 +00:00
Code Issues Releases Activity

kern: support dynamic resource expansion for system heaps/events/sessions.

Browse source
This commit is contained in:
Michael Scire 2021-09-17 22:01:58 -07:00 committed by SciresM
parent 2b91956051
commit 2c4bd44d7e
37 changed files with 856 additions and 328 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
exosphere/program/source/smc/secmon_smc_info.cpp
View file

@ -33,14 +33,15 @@ namespace ams::secmon::smc {
using PhysicalMemorySize = util::BitPack32::Field<16, 2>;
/* Kernel view, from libmesosphere. */
using DebugFillMemory = util::BitPack32::Field<0, 1, bool>;
using EnableUserExceptionHandlers = util::BitPack32::Field<DebugFillMemory::Next, 1, bool>;
using EnableUserPmuAccess = util::BitPack32::Field<EnableUserExceptionHandlers::Next, 1, bool>;
using IncreaseThreadResourceLimit = util::BitPack32::Field<EnableUserPmuAccess::Next, 1, bool>;
using Reserved4 = util::BitPack32::Field<IncreaseThreadResourceLimit::Next, 4, u32>;
using UseSecureMonitorPanicCall = util::BitPack32::Field<Reserved4::Next, 1, bool>;
using Reserved9 = util::BitPack32::Field<UseSecureMonitorPanicCall::Next, 7, u32>;
using MemorySize = util::BitPack32::Field<Reserved9::Next, 2, u32>; /* smc::MemorySize = pkg1::MemorySize */
using DebugFillMemory = util::BitPack32::Field<0, 1, bool>;
using EnableUserExceptionHandlers = util::BitPack32::Field<DebugFillMemory::Next, 1, bool>;
using EnableUserPmuAccess = util::BitPack32::Field<EnableUserExceptionHandlers::Next, 1, bool>;
using IncreaseThreadResourceLimit = util::BitPack32::Field<EnableUserPmuAccess::Next, 1, bool>;
using DisableDynamicResourceLimits = util::BitPack32::Field<IncreaseThreadResourceLimit::Next, 1, bool>;
using Reserved5 = util::BitPack32::Field<DisableDynamicResourceLimits::Next, 3, u32>;
using UseSecureMonitorPanicCall = util::BitPack32::Field<Reserved5::Next, 1, bool>;
using Reserved9 = util::BitPack32::Field<UseSecureMonitorPanicCall::Next, 7, u32>;
using MemorySize = util::BitPack32::Field<Reserved9::Next, 2, u32>; /* smc::MemorySize = pkg1::MemorySize */
};
constexpr const pkg1::MemorySize DramIdToMemorySize[fuse::DramId_Count] = {

31
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp
View file

@ -16,6 +16,7 @@
#pragma once
#include <vapours.hpp>
#include <mesosphere/kern_select_cpu.hpp>
#include <mesosphere/kern_select_interrupt_manager.hpp>
namespace ams::kern::arch::arm64 {
@ -24,6 +25,32 @@ namespace ams::kern::arch::arm64 {
{ t.next } -> std::convertible_to<T *>;
};
ALWAYS_INLINE bool IsSlabAtomicValid() {
/* Without careful consideration, slab heaps atomics are vulnerable to */
/* the ABA problem, when doing compare and swap of node pointers. */
/* We resolve this by using the ARM exclusive monitor; we bundle the */
/* load and store of the relevant values into a single exclusive monitor */
/* hold, preventing the ABA problem. */
/* However, our assembly must do both a load and a store under a single */
/* hold, at different memory addresses. Considering the case where the */
/* addresses are distinct but resolve to the same cache set (by chance), */
/* we can note that under a 1-way associative (direct-mapped) cache */
/* we would have as a guarantee that the second access would evict the */
/* cache line from the first access, invalidating our exclusive monitor */
/* hold. Thus, we require that the cache is not 1-way associative, for */
/* our implementation to be correct. */
{
/* Disable interrupts. */
KScopedInterruptDisable di;
/* Select L1 cache. */
cpu::SetCsselrEl1(0);
/* Check that the L1 cache is not direct-mapped. */
return cpu::CacheSizeIdRegisterAccessor().GetAssociativity() != 0;
}
}
template<typename T> requires SlabHeapNode<T>
ALWAYS_INLINE T *AllocateFromSlabAtomic(T **head) {
u32 tmp;
@ -36,10 +63,7 @@ namespace ams::kern::arch::arm64 {
" ldr %[next], [%[node]]\n"
" stlxr %w[tmp], %[next], [%[head]]\n"
" cbnz %w[tmp], 1b\n"
" b 3f\n"
"2:\n"
" clrex\n"
"3:\n"
: [tmp]"=&r"(tmp), [node]"=&r"(node), [next]"=&r"(next), [head]"+&r"(head)
:
: "cc", "memory"
@ -59,7 +83,6 @@ namespace ams::kern::arch::arm64 {
" str %[next], [%[node]]\n"
" stlxr %w[tmp], %[node], [%[head]]\n"
" cbnz %w[tmp], 1b\n"
"2:\n"
: [tmp]"=&r"(tmp), [node]"+&r"(node), [next]"=&r"(next), [head]"+&r"(head)
:
: "cc", "memory"

60
libraries/libmesosphere/include/mesosphere/kern_k_dynamic_resource_manager.hpp Normal file
View file

@ -0,0 +1,60 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
namespace ams::kern {
template<typename T, bool ClearNode = false>
class KDynamicResourceManager {
NON_COPYABLE(KDynamicResourceManager);
NON_MOVEABLE(KDynamicResourceManager);
public:
using DynamicSlabType = KDynamicSlabHeap<T, ClearNode>;
private:
KDynamicPageManager *m_page_allocator{};
DynamicSlabType *m_slab_heap{};
public:
constexpr KDynamicResourceManager() = default;
constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_slab_heap->GetAddress(); }
constexpr ALWAYS_INLINE size_t GetSize() const { return m_slab_heap->GetSize(); }
constexpr ALWAYS_INLINE size_t GetUsed() const { return m_slab_heap->GetUsed(); }
constexpr ALWAYS_INLINE size_t GetPeak() const { return m_slab_heap->GetPeak(); }
constexpr ALWAYS_INLINE size_t GetCount() const { return m_slab_heap->GetCount(); }
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, DynamicSlabType *slab_heap) {
m_page_allocator = page_allocator;
m_slab_heap = slab_heap;
}
T *Allocate() const {
return m_slab_heap->Allocate(m_page_allocator);
}
void Free(T *t) const {
m_slab_heap->Free(t);
}
};
class KBlockInfoManager : public KDynamicResourceManager<KBlockInfo>{};
class KMemoryBlockSlabManager : public KDynamicResourceManager<KMemoryBlock>{};
using KBlockInfoSlabHeap = typename KBlockInfoManager::DynamicSlabType;
using KMemoryBlockSlabHeap = typename KMemoryBlockSlabManager::DynamicSlabType;
}

91
libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp
View file

@ -23,95 +23,71 @@
namespace ams::kern {
template<typename T, bool ClearNode = false>
class KDynamicSlabHeap {
class KDynamicSlabHeap : protected impl::KSlabHeapImpl {
NON_COPYABLE(KDynamicSlabHeap);
NON_MOVEABLE(KDynamicSlabHeap);
private:
using Impl = impl::KSlabHeapImpl;
using PageBuffer = KDynamicPageManager::PageBuffer;
private:
Impl m_impl;
KDynamicPageManager *m_page_allocator;
std::atomic<size_t> m_used;
std::atomic<size_t> m_peak;
std::atomic<size_t> m_count;
KVirtualAddress m_address;
size_t m_size;
private:
ALWAYS_INLINE Impl *GetImpl() {
return std::addressof(m_impl);
}
ALWAYS_INLINE const Impl *GetImpl() const {
return std::addressof(m_impl);
}
std::atomic<size_t> m_used{};
std::atomic<size_t> m_peak{};
std::atomic<size_t> m_count{};
KVirtualAddress m_address{};
size_t m_size{};
public:
constexpr KDynamicSlabHeap() : m_impl(), m_page_allocator(), m_used(), m_peak(), m_count(), m_address(), m_size() { /* ... */ }
constexpr KDynamicSlabHeap() = default;
constexpr KVirtualAddress GetAddress() const { return m_address; }
constexpr size_t GetSize() const { return m_size; }
constexpr size_t GetUsed() const { return m_used.load(); }
constexpr size_t GetPeak() const { return m_peak.load(); }
constexpr size_t GetCount() const { return m_count.load(); }
constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_address; }
constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.load(); }
constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.load(); }
constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.load(); }
constexpr bool IsInRange(KVirtualAddress addr) const {
constexpr ALWAYS_INLINE bool IsInRange(KVirtualAddress addr) const {
return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1;
}
void Initialize(KVirtualAddress memory, size_t sz) {
/* Set tracking fields. */
m_address = memory;
m_count = sz / sizeof(T);
m_size = m_count * sizeof(T);
/* Free blocks to memory. */
u8 *cur = GetPointer<u8>(m_address + m_size);
for (size_t i = 0; i < sz / sizeof(T); i++) {
cur -= sizeof(T);
this->GetImpl()->Free(cur);
}
}
void Initialize(KDynamicPageManager *page_allocator) {
m_page_allocator = page_allocator;
m_address = m_page_allocator->GetAddress();
m_size = m_page_allocator->GetSize();
}
void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
MESOSPHERE_ASSERT(page_allocator != nullptr);
/* Initialize members. */
this->Initialize(page_allocator);
m_address = page_allocator->GetAddress();
m_size = page_allocator->GetSize();
/* Initialize the base allocator. */
KSlabHeapImpl::Initialize();
/* Allocate until we have the correct number of objects. */
while (m_count.load() < num_objects) {
auto *allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
auto *allocated = reinterpret_cast<T *>(page_allocator->Allocate());
MESOSPHERE_ABORT_UNLESS(allocated != nullptr);
for (size_t i = 0; i < sizeof(PageBuffer) / sizeof(T); i++) {
this->GetImpl()->Free(allocated + i);
KSlabHeapImpl::Free(allocated + i);
}
m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
}
}
T *Allocate() {
T *allocated = reinterpret_cast<T *>(this->GetImpl()->Allocate());
ALWAYS_INLINE T *Allocate(KDynamicPageManager *page_allocator) {
T *allocated = static_cast<T *>(KSlabHeapImpl::Allocate());
/* If we successfully allocated and we should clear the node, do so. */
if constexpr (ClearNode) {
if (AMS_LIKELY(allocated != nullptr)) {
reinterpret_cast<Impl::Node *>(allocated)->next = nullptr;
reinterpret_cast<KSlabHeapImpl::Node *>(allocated)->next = nullptr;
}
}
/* If we fail to allocate, try to get a new page from our next allocator. */
if (AMS_UNLIKELY(allocated == nullptr)) {
if (m_page_allocator != nullptr) {
allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
if (AMS_UNLIKELY(allocated == nullptr) ) {
if (page_allocator != nullptr) {
allocated = reinterpret_cast<T *>(page_allocator->Allocate());
if (allocated != nullptr) {
/* If we succeeded in getting a page, free the rest to our slab. */
for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) {
this->GetImpl()->Free(allocated + i);
KSlabHeapImpl::Free(allocated + i);
}
m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
}
@ -135,13 +111,10 @@ namespace ams::kern {
return allocated;
}
void Free(T *t) {
this->GetImpl()->Free(t);
ALWAYS_INLINE void Free(T *t) {
KSlabHeapImpl::Free(t);
m_used.fetch_sub(1);
}
};
class KBlockInfoManager : public KDynamicSlabHeap<KBlockInfo>{};
class KMemoryBlockSlabManager : public KDynamicSlabHeap<KMemoryBlock>{};
}

2
libraries/libmesosphere/include/mesosphere/kern_k_event.hpp
View file

@ -21,7 +21,7 @@
namespace ams::kern {
class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList> {
class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList, true> {
MESOSPHERE_AUTOOBJECT_TRAITS(KEvent, KAutoObject);
private:
KReadableEvent m_readable_event;

2
libraries/libmesosphere/include/mesosphere/kern_k_light_session.hpp
View file

@ -25,7 +25,7 @@ namespace ams::kern {
class KClientPort;
class KProcess;
class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList> {
class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList, true> {
MESOSPHERE_AUTOOBJECT_TRAITS(KLightSession, KAutoObject);
private:
enum class State : u8 {

1
libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp
View file

@ -144,6 +144,7 @@ namespace ams::kern {
static NOINLINE const KMemoryRegion &GetPageTableHeapRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); }
static NOINLINE const KMemoryRegion &GetKernelStackRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); }
static NOINLINE const KMemoryRegion &GetTempRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); }
static NOINLINE const KMemoryRegion &GetSlabRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)); }
static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); }

83
libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp
View file

@ -15,58 +15,27 @@
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_slab_helpers.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
#include <mesosphere/kern_k_page_table_slab_heap.hpp>
#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
namespace ams::kern {
namespace impl {
class PageTablePage {
private:
u8 m_buffer[PageSize];
public:
ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
};
static_assert(sizeof(PageTablePage) == PageSize);
}
class KPageTableManager : public KDynamicSlabHeap<impl::PageTablePage, true> {
class KPageTableManager : public KDynamicResourceManager<impl::PageTablePage, true> {
public:
using RefCount = u16;
static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
static_assert(PageTableSize == PageSize);
using RefCount = KPageTableSlabHeap::RefCount;
static constexpr size_t PageTableSize = KPageTableSlabHeap::PageTableSize;
private:
using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
using BaseHeap = KDynamicResourceManager<impl::PageTablePage, true>;
private:
RefCount *m_ref_counts;
KPageTableSlabHeap *m_pt_heap{};
public:
static constexpr size_t CalculateReferenceCountSize(size_t size) {
return (size / PageSize) * sizeof(RefCount);
}
public:
constexpr KPageTableManager() : BaseHeap(), m_ref_counts() { /* ... */ }
private:
void Initialize(RefCount *rc) {
m_ref_counts = rc;
for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
m_ref_counts[i] = 0;
}
}
constexpr KPageTableManager() = default;
constexpr RefCount *GetRefCountPointer(KVirtualAddress addr) const {
return std::addressof(m_ref_counts[(addr - this->GetAddress()) / PageSize]);
}
public:
void Initialize(KDynamicPageManager *page_allocator, RefCount *rc) {
BaseHeap::Initialize(page_allocator);
this->Initialize(rc);
}
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, KPageTableSlabHeap *pt_heap) {
m_pt_heap = pt_heap;
void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
BaseHeap::Initialize(page_allocator, object_count);
this->Initialize(rc);
static_assert(std::derived_from<KPageTableSlabHeap, DynamicSlabType>);
BaseHeap::Initialize(page_allocator, pt_heap);
}
KVirtualAddress Allocate() {
@ -74,33 +43,23 @@ namespace ams::kern {
}
void Free(KVirtualAddress addr) {
/* Free the page. */
BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
return BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
}
RefCount GetRefCount(KVirtualAddress addr) const {
MESOSPHERE_ASSERT(this->IsInRange(addr));
return *this->GetRefCountPointer(addr);
ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
return m_pt_heap->GetRefCount(addr);
}
void Open(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
*this->GetRefCountPointer(addr) += count;
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
return m_pt_heap->Open(addr, count);
}
bool Close(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
*this->GetRefCountPointer(addr) -= count;
return this->GetRefCount(addr) == 0;
ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
return m_pt_heap->Close(addr, count);
}
constexpr bool IsInPageTableHeap(KVirtualAddress addr) const {
return this->IsInRange(addr);
constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
return m_pt_heap->IsInRange(addr);
}
};

93
libraries/libmesosphere/include/mesosphere/kern_k_page_table_slab_heap.hpp Normal file
View file

@ -0,0 +1,93 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_slab_helpers.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
namespace ams::kern {
namespace impl {
class PageTablePage {
private:
u8 m_buffer[PageSize];
public:
ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
};
static_assert(sizeof(PageTablePage) == PageSize);
}
class KPageTableSlabHeap : public KDynamicSlabHeap<impl::PageTablePage, true> {
public:
using RefCount = u16;
static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
static_assert(PageTableSize == PageSize);
private:
using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
private:
RefCount *m_ref_counts{};
public:
static constexpr ALWAYS_INLINE size_t CalculateReferenceCountSize(size_t size) {
return (size / PageSize) * sizeof(RefCount);
}
public:
constexpr KPageTableSlabHeap() = default;
private:
ALWAYS_INLINE void Initialize(RefCount *rc) {
m_ref_counts = rc;
for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
m_ref_counts[i] = 0;
}
}
constexpr ALWAYS_INLINE RefCount *GetRefCountPointer(KVirtualAddress addr) const {
return m_ref_counts + ((addr - this->GetAddress()) / PageSize);
}
public:
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
BaseHeap::Initialize(page_allocator, object_count);
this->Initialize(rc);
}
ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
MESOSPHERE_ASSERT(this->IsInRange(addr));
return *this->GetRefCountPointer(addr);
}
ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
*this->GetRefCountPointer(addr) += count;
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
}
ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
*this->GetRefCountPointer(addr) -= count;
return this->GetRefCount(addr) == 0;
}
constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
return this->IsInRange(addr);
}
};
}

5
libraries/libmesosphere/include/mesosphere/kern_k_process.hpp
View file

@ -29,7 +29,7 @@
#include <mesosphere/kern_k_address_arbiter.hpp>
#include <mesosphere/kern_k_capabilities.hpp>
#include <mesosphere/kern_k_wait_object.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
#include <mesosphere/kern_k_page_table_manager.hpp>
namespace ams::kern {
@ -121,6 +121,9 @@ namespace ams::kern {
KMemoryBlockSlabManager m_memory_block_slab_manager{};
KBlockInfoManager m_block_info_manager{};
KPageTableManager m_page_table_manager{};
KMemoryBlockSlabHeap m_memory_block_heap{};
KBlockInfoSlabHeap m_block_info_heap{};
KPageTableSlabHeap m_page_table_heap{};
private:
Result Initialize(const ams::svc::CreateProcessParameter &params);

2
libraries/libmesosphere/include/mesosphere/kern_k_resource_limit.hpp
View file

@ -47,6 +47,8 @@ namespace ams::kern {
Result SetLimitValue(ams::svc::LimitableResource which, s64 value);
void Add(ams::svc::LimitableResource which, s64 value);
bool Reserve(ams::svc::LimitableResource which, s64 value);
bool Reserve(ams::svc::LimitableResource which, s64 value, s64 timeout);
void Release(ams::svc::LimitableResource which, s64 value);

2
libraries/libmesosphere/include/mesosphere/kern_k_session.hpp
View file

@ -25,7 +25,7 @@ namespace ams::kern {
class KClientPort;
class KProcess;
class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList> {
class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList, true> {
MESOSPHERE_AUTOOBJECT_TRAITS(KSession, KAutoObject);
private:
enum class State : u8 {

10
libraries/libmesosphere/include/mesosphere/kern_k_session_request.hpp
View file

@ -24,7 +24,7 @@
namespace ams::kern {
class KSessionRequest final : public KSlabAllocated<KSessionRequest>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
class KSessionRequest final : public KSlabAllocated<KSessionRequest, true>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
MESOSPHERE_AUTOOBJECT_TRAITS(KSessionRequest, KAutoObject);
public:
class SessionMappings {
@ -140,6 +140,14 @@ namespace ams::kern {
return req;
}
static KSessionRequest *CreateFromUnusedSlabMemory() {
KSessionRequest *req = KSessionRequest::AllocateFromUnusedSlabMemory();
if (req != nullptr) {
KAutoObject::Create(req);
}
return req;
}
virtual void Destroy() override {
this->Finalize();
KSessionRequest::Free(this);

2
libraries/libmesosphere/include/mesosphere/kern_k_shared_memory.hpp
View file

@ -35,7 +35,7 @@ namespace ams::kern {
bool m_is_initialized;
public:
explicit KSharedMemory()
: m_page_group(std::addressof(Kernel::GetBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
: m_page_group(std::addressof(Kernel::GetSystemBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
m_owner_perm(ams::svc::MemoryPermission_None), m_remote_perm(ams::svc::MemoryPermission_None), m_is_initialized(false)
{
/* ... */

173
libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp
View file

@ -16,11 +16,13 @@
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_typed_address.hpp>
#include <mesosphere/kern_k_memory_layout.hpp>
#if defined(ATMOSPHERE_ARCH_ARM64)
#include <mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp>
namespace ams::kern {
using ams::kern::arch::arm64::IsSlabAtomicValid;
using ams::kern::arch::arm64::AllocateFromSlabAtomic;
using ams::kern::arch::arm64::FreeToSlabAtomic;
}
@ -44,78 +46,73 @@ namespace ams::kern {
Node *next;
};
private:
Node * m_head;
size_t m_obj_size;
Node *m_head{nullptr};
public:
constexpr KSlabHeapImpl() : m_head(nullptr), m_obj_size(0) { MESOSPHERE_ASSERT_THIS(); }
constexpr KSlabHeapImpl() = default;
void Initialize(size_t size) {
MESOSPHERE_INIT_ABORT_UNLESS(m_head == nullptr);
m_obj_size = size;
void Initialize() {
MESOSPHERE_ABORT_UNLESS(m_head == nullptr);
MESOSPHERE_ABORT_UNLESS(IsSlabAtomicValid());
}
Node *GetHead() const {
ALWAYS_INLINE Node *GetHead() const {
return m_head;
}
size_t GetObjectSize() const {
return m_obj_size;
}
void *Allocate() {
MESOSPHERE_ASSERT_THIS();
ALWAYS_INLINE void *Allocate() {
return AllocateFromSlabAtomic(std::addressof(m_head));
}
void Free(void *obj) {
MESOSPHERE_ASSERT_THIS();
Node *node = reinterpret_cast<Node *>(obj);
return FreeToSlabAtomic(std::addressof(m_head), node);
ALWAYS_INLINE void Free(void *obj) {
return FreeToSlabAtomic(std::addressof(m_head), static_cast<Node *>(obj));
}
};
}
class KSlabHeapBase {
template<bool SupportDynamicExpansion>
class KSlabHeapBase : protected impl::KSlabHeapImpl {
NON_COPYABLE(KSlabHeapBase);
NON_MOVEABLE(KSlabHeapBase);
private:
using Impl = impl::KSlabHeapImpl;
size_t m_obj_size{};
uintptr_t m_peak{};
uintptr_t m_start{};
uintptr_t m_end{};
private:
Impl m_impl;
uintptr_t m_peak;
uintptr_t m_start;
uintptr_t m_end;
private:
ALWAYS_INLINE Impl *GetImpl() {
return std::addressof(m_impl);
}
ALWAYS_INLINE const Impl *GetImpl() const {
return std::addressof(m_impl);
ALWAYS_INLINE void UpdatePeakImpl(uintptr_t obj) {
static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
std::atomic_ref<uintptr_t> peak_ref(m_peak);
const uintptr_t alloc_peak = obj + this->GetObjectSize();
uintptr_t cur_peak = m_peak;
do {
if (alloc_peak <= cur_peak) {
break;
}
} while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
}
public:
constexpr KSlabHeapBase() : m_impl(), m_peak(0), m_start(0), m_end(0) { MESOSPHERE_ASSERT_THIS(); }
constexpr KSlabHeapBase() = default;
ALWAYS_INLINE bool Contains(uintptr_t address) const {
return m_start <= address && address < m_end;
}
void InitializeImpl(size_t obj_size, void *memory, size_t memory_size) {
MESOSPHERE_ASSERT_THIS();
void Initialize(size_t obj_size, void *memory, size_t memory_size) {
/* Ensure we don't initialize a slab using null memory. */
MESOSPHERE_ABORT_UNLESS(memory != nullptr);
/* Set our object size. */
m_obj_size = obj_size;
/* Initialize the base allocator. */
this->GetImpl()->Initialize(obj_size);
KSlabHeapImpl::Initialize();
/* Set our tracking variables. */
const size_t num_obj = (memory_size / obj_size);
m_start = reinterpret_cast<uintptr_t>(memory);
m_end = m_start + num_obj * obj_size;
m_end = m_start + num_obj * obj_size;
m_peak = m_start;
/* Free the objects. */
@ -123,75 +120,91 @@ namespace ams::kern {
for (size_t i = 0; i < num_obj; i++) {
cur -= obj_size;
this->GetImpl()->Free(cur);
KSlabHeapImpl::Free(cur);
}
}
size_t GetSlabHeapSize() const {
ALWAYS_INLINE size_t GetSlabHeapSize() const {
return (m_end - m_start) / this->GetObjectSize();
}
size_t GetObjectSize() const {
return this->GetImpl()->GetObjectSize();
ALWAYS_INLINE size_t GetObjectSize() const {
return m_obj_size;
}
void *AllocateImpl() {
MESOSPHERE_ASSERT_THIS();
void *obj = this->GetImpl()->Allocate();
ALWAYS_INLINE void *Allocate() {
void *obj = KSlabHeapImpl::Allocate();
/* Track the allocated peak. */
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
if (AMS_LIKELY(obj != nullptr)) {
static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
std::atomic_ref<uintptr_t> peak_ref(m_peak);
const uintptr_t alloc_peak = reinterpret_cast<uintptr_t>(obj) + this->GetObjectSize();
uintptr_t cur_peak = m_peak;
do {
if (alloc_peak <= cur_peak) {
break;
if constexpr (SupportDynamicExpansion) {
if (this->Contains(reinterpret_cast<uintptr_t>(obj))) {
this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
} else {
this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(m_end) - this->GetObjectSize());
}
} while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
} else {
this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
}
}
#endif
return obj;
}
void FreeImpl(void *obj) {
MESOSPHERE_ASSERT_THIS();
ALWAYS_INLINE void Free(void *obj) {
/* Don't allow freeing an object that wasn't allocated from this heap. */
MESOSPHERE_ABORT_UNLESS(this->Contains(reinterpret_cast<uintptr_t>(obj)));
const bool contained = this->Contains(reinterpret_cast<uintptr_t>(obj));
if constexpr (SupportDynamicExpansion) {
const bool is_slab = KMemoryLayout::GetSlabRegion().Contains(reinterpret_cast<uintptr_t>(obj));
MESOSPHERE_ABORT_UNLESS(contained || is_slab);
} else {
MESOSPHERE_ABORT_UNLESS(contained);
}
this->GetImpl()->Free(obj);
KSlabHeapImpl::Free(obj);
}
size_t GetObjectIndexImpl(const void *obj) const {
ALWAYS_INLINE size_t GetObjectIndex(const void *obj) const {
if constexpr (SupportDynamicExpansion) {
if (!this->Contains(reinterpret_cast<uintptr_t>(obj))) {
return std::numeric_limits<size_t>::max();
}
}
return (reinterpret_cast<uintptr_t>(obj) - m_start) / this->GetObjectSize();
}
size_t GetPeakIndex() const {
return this->GetObjectIndexImpl(reinterpret_cast<const void *>(m_peak));
ALWAYS_INLINE size_t GetPeakIndex() const {
return this->GetObjectIndex(reinterpret_cast<const void *>(m_peak));
}
uintptr_t GetSlabHeapAddress() const {
ALWAYS_INLINE uintptr_t GetSlabHeapAddress() const {
return m_start;
}
size_t GetNumRemaining() const {
ALWAYS_INLINE size_t GetNumRemaining() const {
size_t remaining = 0;
/* Only calculate the number of remaining objects under debug configuration. */
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
while (true) {
auto *cur = this->GetImpl()->GetHead();
auto *cur = this->GetHead();
remaining = 0;
while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
++remaining;
cur = cur->next;
if constexpr (SupportDynamicExpansion) {
const auto &slab_region = KMemoryLayout::GetSlabRegion();
while (this->Contains(reinterpret_cast<uintptr_t>(cur)) || slab_region.Contains(reinterpret_cast<uintptr_t>(cur))) {
++remaining;
cur = cur->next;
}
} else {
while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
++remaining;
cur = cur->next;
}
}
if (cur == nullptr) {
@ -204,29 +217,31 @@ namespace ams::kern {
}
};
template<typename T>
class KSlabHeap : public KSlabHeapBase {
template<typename T, bool SupportDynamicExpansion>
class KSlabHeap : public KSlabHeapBase<SupportDynamicExpansion> {
private:
using BaseHeap = KSlabHeapBase<SupportDynamicExpansion>;
public:
constexpr KSlabHeap() : KSlabHeapBase() { /* ... */ }
constexpr KSlabHeap() = default;
void Initialize(void *memory, size_t memory_size) {
this->InitializeImpl(sizeof(T), memory, memory_size);
BaseHeap::Initialize(sizeof(T), memory, memory_size);
}
T *Allocate() {
T *obj = reinterpret_cast<T *>(this->AllocateImpl());
ALWAYS_INLINE T *Allocate() {
T *obj = static_cast<T *>(BaseHeap::Allocate());
if (AMS_LIKELY(obj != nullptr)) {
std::construct_at(obj);
}
return obj;
}
void Free(T *obj) {
this->FreeImpl(obj);
ALWAYS_INLINE void Free(T *obj) {
BaseHeap::Free(obj);
}
size_t GetObjectIndex(const T *obj) const {
return this->GetObjectIndexImpl(obj);
ALWAYS_INLINE size_t GetObjectIndex(const T *obj) const {
return BaseHeap::GetObjectIndex(obj);
}
};

15
libraries/libmesosphere/include/mesosphere/kern_k_target_system.hpp
View file

@ -23,12 +23,13 @@ namespace ams::kern {
private:
friend class KSystemControl;
private:
static inline bool s_is_debug_mode;
static inline bool s_enable_debug_logging;
static inline bool s_enable_user_exception_handlers;
static inline bool s_enable_debug_memory_fill;
static inline bool s_enable_user_pmu_access;
static inline bool s_enable_kernel_debugging;
static inline constinit bool s_is_debug_mode;
static inline constinit bool s_enable_debug_logging;
static inline constinit bool s_enable_user_exception_handlers;
static inline constinit bool s_enable_debug_memory_fill;
static inline constinit bool s_enable_user_pmu_access;
static inline constinit bool s_enable_kernel_debugging;
static inline constinit bool s_enable_dynamic_resource_limits;
private:
static ALWAYS_INLINE void SetIsDebugMode(bool en) { s_is_debug_mode = en; }
static ALWAYS_INLINE void EnableDebugLogging(bool en) { s_enable_debug_logging = en; }
@ -36,6 +37,7 @@ namespace ams::kern {
static ALWAYS_INLINE void EnableDebugMemoryFill(bool en) { s_enable_debug_memory_fill = en; }
static ALWAYS_INLINE void EnableUserPmuAccess(bool en) { s_enable_user_pmu_access = en; }
static ALWAYS_INLINE void EnableKernelDebugging(bool en) { s_enable_kernel_debugging = en; }
static ALWAYS_INLINE void EnableDynamicResourceLimits(bool en) { s_enable_dynamic_resource_limits = en; }
public:
static ALWAYS_INLINE bool IsDebugMode() { return s_is_debug_mode; }
static ALWAYS_INLINE bool IsDebugLoggingEnabled() { return s_enable_debug_logging; }
@ -43,6 +45,7 @@ namespace ams::kern {
static ALWAYS_INLINE bool IsDebugMemoryFillEnabled() { return s_enable_debug_memory_fill; }
static ALWAYS_INLINE bool IsUserPmuAccessEnabled() { return s_enable_user_pmu_access; }
static ALWAYS_INLINE bool IsKernelDebuggingEnabled() { return s_enable_kernel_debugging; }
static ALWAYS_INLINE bool IsDynamicResourceLimitsEnabled() { return s_enable_dynamic_resource_limits; }
};
}

27
libraries/libmesosphere/include/mesosphere/kern_k_unused_slab_memory.hpp Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_typed_address.hpp>
namespace ams::kern {
/* Utilities to allocate/free memory from the "unused" gaps between slab heaps. */
/* See KTargetSystem::IsDynamicResourceLimitsEnabled() usage for more context. */
KVirtualAddress AllocateUnusedSlabMemory(size_t size, size_t alignment);
void FreeUnusedSlabMemory(KVirtualAddress address, size_t size);
}

27
libraries/libmesosphere/include/mesosphere/kern_kernel.hpp
View file

@ -63,14 +63,21 @@ namespace ams::kern {
static constexpr size_t ApplicationMemoryBlockSlabHeapSize = 20000;
static constexpr size_t SystemMemoryBlockSlabHeapSize = 10000;
static constexpr size_t BlockInfoSlabHeapSize = 4000;
static constexpr size_t ReservedDynamicPageCount = 70;
private:
static State s_state;
static KResourceLimit s_system_resource_limit;
static KMemoryManager s_memory_manager;
static KPageTableManager s_page_table_manager;
static KPageTableSlabHeap s_page_table_heap;
static KMemoryBlockSlabHeap s_app_memory_block_heap;
static KMemoryBlockSlabHeap s_sys_memory_block_heap;
static KBlockInfoSlabHeap s_block_info_heap;
static KPageTableManager s_app_page_table_manager;
static KPageTableManager s_sys_page_table_manager;
static KMemoryBlockSlabManager s_app_memory_block_manager;
static KMemoryBlockSlabManager s_sys_memory_block_manager;
static KBlockInfoManager s_block_info_manager;
static KBlockInfoManager s_app_block_info_manager;
static KBlockInfoManager s_sys_block_info_manager;
static KSupervisorPageTable s_supervisor_page_table;
static KUnsafeMemory s_unsafe_memory;
static KWorkerTaskManager s_worker_task_managers[KWorkerTaskManager::WorkerType_Count];
@ -130,12 +137,20 @@ namespace ams::kern {
return s_sys_memory_block_manager;
}
static ALWAYS_INLINE KBlockInfoManager &GetBlockInfoManager() {
return s_block_info_manager;
static ALWAYS_INLINE KBlockInfoManager &GetApplicationBlockInfoManager() {
return s_app_block_info_manager;
}
static ALWAYS_INLINE KPageTableManager &GetPageTableManager() {
return s_page_table_manager;
static ALWAYS_INLINE KBlockInfoManager &GetSystemBlockInfoManager() {
return s_sys_block_info_manager;
}
static ALWAYS_INLINE KPageTableManager &GetApplicationPageTableManager() {
return s_app_page_table_manager;
}
static ALWAYS_INLINE KPageTableManager &GetSystemPageTableManager() {
return s_sys_page_table_manager;
}
static ALWAYS_INLINE KSupervisorPageTable &GetKernelPageTable() {

42
libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp
View file

@ -18,15 +18,16 @@
#include <mesosphere/kern_k_auto_object.hpp>
#include <mesosphere/kern_k_slab_heap.hpp>
#include <mesosphere/kern_k_auto_object_container.hpp>
#include <mesosphere/kern_k_unused_slab_memory.hpp>
namespace ams::kern {
template<class Derived>
template<class Derived, bool SupportDynamicExpansion = false>
class KSlabAllocated {
private:
static inline KSlabHeap<Derived> s_slab_heap;
static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
public:
constexpr KSlabAllocated() { /* ... */ }
constexpr KSlabAllocated() = default;
size_t GetSlabIndex() const {
return s_slab_heap.GetIndex(static_cast<const Derived *>(this));
@ -36,14 +37,25 @@ namespace ams::kern {
s_slab_heap.Initialize(memory, memory_size);
}
static ALWAYS_INLINE Derived *Allocate() {
static Derived *Allocate() {
return s_slab_heap.Allocate();
}
static ALWAYS_INLINE void Free(Derived *obj) {
static void Free(Derived *obj) {
s_slab_heap.Free(obj);
}
template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
static Derived *AllocateFromUnusedSlabMemory() {
static_assert(Enable == SupportDynamicExpansion);
Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
if (AMS_LIKELY(obj != nullptr)) {
std::construct_at(obj);
}
return obj;
}
static size_t GetObjectSize() { return s_slab_heap.GetObjectSize(); }
static size_t GetSlabHeapSize() { return s_slab_heap.GetSlabHeapSize(); }
static size_t GetPeakIndex() { return s_slab_heap.GetPeakIndex(); }
@ -52,12 +64,12 @@ namespace ams::kern {
static size_t GetNumRemaining() { return s_slab_heap.GetNumRemaining(); }
};
template<typename Derived, typename Base>
template<typename Derived, typename Base, bool SupportDynamicExpansion = false>
class KAutoObjectWithSlabHeapAndContainer : public Base {
static_assert(std::is_base_of<KAutoObjectWithList, Base>::value);
private:
static inline KSlabHeap<Derived> s_slab_heap;
static inline KAutoObjectWithListContainer s_container;
static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
static constinit inline KAutoObjectWithListContainer s_container;
private:
static ALWAYS_INLINE Derived *Allocate() {
return s_slab_heap.Allocate();
@ -73,7 +85,7 @@ namespace ams::kern {
ALWAYS_INLINE ~ListAccessor() { /* ... */ }
};
public:
constexpr KAutoObjectWithSlabHeapAndContainer() : Base() { /* ... */ }
constexpr KAutoObjectWithSlabHeapAndContainer() = default;
virtual void Destroy() override {
const bool is_initialized = this->IsInitialized();
@ -109,6 +121,18 @@ namespace ams::kern {
return obj;
}
template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
static Derived *CreateFromUnusedSlabMemory() {
static_assert(Enable == SupportDynamicExpansion);
Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
if (AMS_LIKELY(obj != nullptr)) {
std::construct_at(obj);
KAutoObject::Create(obj);
}
return obj;
}
static void Register(Derived *obj) {
return s_container.Register(obj);
}

2
libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp
View file

@ -166,7 +166,7 @@ namespace ams::kern::arch::arm64 {
Result KPageTable::InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end) {
/* Initialize basic fields. */
m_asid = 0;
m_manager = std::addressof(Kernel::GetPageTableManager());
m_manager = std::addressof(Kernel::GetSystemPageTableManager());
/* Allocate a page for ttbr. */
const u64 asid_tag = (static_cast<u64>(m_asid) << 48ul);

14
libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp
View file

@ -650,11 +650,11 @@ namespace ams::kern::board::nintendo::nx {
g_memory_controller_address = KMemoryLayout::GetDevicePhysicalAddress(KMemoryRegionType_MemoryController);
/* Allocate a page to use as a reserved/no device table. */
const KVirtualAddress table_virt_addr = Kernel::GetPageTableManager().Allocate();
const KVirtualAddress table_virt_addr = Kernel::GetSystemPageTableManager().Allocate();
MESOSPHERE_ABORT_UNLESS(table_virt_addr != Null<KVirtualAddress>);
const KPhysicalAddress table_phys_addr = GetPageTablePhysicalAddress(table_virt_addr);
MESOSPHERE_ASSERT(IsValidPhysicalAddress(table_phys_addr));
Kernel::GetPageTableManager().Open(table_virt_addr, 1);
Kernel::GetSystemPageTableManager().Open(table_virt_addr, 1);
/* Clear the page and save it. */
/* NOTE: Nintendo does not check the result of StoreDataCache. */
@ -779,7 +779,7 @@ namespace ams::kern::board::nintendo::nx {
const size_t end_index = (space_address + space_size - 1) / DeviceRegionSize;
/* Get the page table manager. */
auto &ptm = Kernel::GetPageTableManager();
auto &ptm = Kernel::GetSystemPageTableManager();
/* Clear the tables. */
static_assert(TableCount == (1ul << DeviceVirtualAddressBits) / DeviceRegionSize);
@ -840,7 +840,7 @@ namespace ams::kern::board::nintendo::nx {
void KDevicePageTable::Finalize() {
/* Get the page table manager. */
auto &ptm = Kernel::GetPageTableManager();
auto &ptm = Kernel::GetSystemPageTableManager();
/* Detach from all devices. */
{
@ -1017,7 +1017,7 @@ namespace ams::kern::board::nintendo::nx {
/* Get the memory manager and page table manager. */
KMemoryManager &mm = Kernel::GetMemoryManager();
KPageTableManager &ptm = Kernel::GetPageTableManager();
KPageTableManager &ptm = Kernel::GetSystemPageTableManager();
/* Cache permissions. */
const bool read = (device_perm & ams::svc::MemoryPermission_Read) != 0;
@ -1181,10 +1181,10 @@ namespace ams::kern::board::nintendo::nx {
/* Get the memory manager and page table manager. */
KMemoryManager &mm = Kernel::GetMemoryManager();
KPageTableManager &ptm = Kernel::GetPageTableManager();
KPageTableManager &ptm = Kernel::GetSystemPageTableManager();
/* Make a page group for the pages we're closing. */
KPageGroup pg(std::addressof(Kernel::GetBlockInfoManager()));
KPageGroup pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
/* Walk the directory. */
u64 remaining = size;

1
libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp
View file

@ -459,6 +459,7 @@ namespace ams::kern::board::nintendo::nx {
KTargetSystem::EnableDebugMemoryFill(kernel_config.Get<smc::KernelConfiguration::DebugFillMemory>());
KTargetSystem::EnableUserExceptionHandlers(kernel_config.Get<smc::KernelConfiguration::EnableUserExceptionHandlers>());
KTargetSystem::EnableDynamicResourceLimits(!kernel_config.Get<smc::KernelConfiguration::DisableDynamicResourceLimits>());
KTargetSystem::EnableUserPmuAccess(kernel_config.Get<smc::KernelConfiguration::EnableUserPmuAccess>());
g_call_smc_on_panic = kernel_config.Get<smc::KernelConfiguration::UseSecureMonitorPanicCall>();

17
libraries/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.hpp
View file

@ -80,14 +80,15 @@ namespace ams::kern::board::nintendo::nx::smc {
};
struct KernelConfiguration {
using DebugFillMemory = util::BitPack32::Field<0, 1, bool>;
using EnableUserExceptionHandlers = util::BitPack32::Field<DebugFillMemory::Next, 1, bool>;
using EnableUserPmuAccess = util::BitPack32::Field<EnableUserExceptionHandlers::Next, 1, bool>;
using IncreaseThreadResourceLimit = util::BitPack32::Field<EnableUserPmuAccess::Next, 1, bool>;
using Reserved4 = util::BitPack32::Field<IncreaseThreadResourceLimit::Next, 4, u32>;
using UseSecureMonitorPanicCall = util::BitPack32::Field<Reserved4::Next, 1, bool>;
using Reserved9 = util::BitPack32::Field<UseSecureMonitorPanicCall::Next, 7, u32>;
using MemorySize = util::BitPack32::Field<Reserved9::Next, 2, smc::MemorySize>;
using DebugFillMemory = util::BitPack32::Field<0, 1, bool>;
using EnableUserExceptionHandlers = util::BitPack32::Field<DebugFillMemory::Next, 1, bool>;
using EnableUserPmuAccess = util::BitPack32::Field<EnableUserExceptionHandlers::Next, 1, bool>;
using IncreaseThreadResourceLimit = util::BitPack32::Field<EnableUserPmuAccess::Next, 1, bool>;
using DisableDynamicResourceLimits = util::BitPack32::Field<IncreaseThreadResourceLimit::Next, 1, bool>;
using Reserved5 = util::BitPack32::Field<DisableDynamicResourceLimits::Next, 3, u32>;
using UseSecureMonitorPanicCall = util::BitPack32::Field<Reserved5::Next, 1, bool>;
using Reserved9 = util::BitPack32::Field<UseSecureMonitorPanicCall::Next, 7, u32>;
using MemorySize = util::BitPack32::Field<Reserved9::Next, 2, smc::MemorySize>;
};
enum UserRebootType {

31
libraries/libmesosphere/source/init/kern_init_slab_setup.cpp
View file

@ -173,8 +173,9 @@ namespace ams::kern::init {
}
void InitializeSlabHeaps() {
/* Get the start of the slab region, since that's where we'll be working. */
KVirtualAddress address = KMemoryLayout::GetSlabRegionAddress();
/* Get the slab region, since that's where we'll be working. */
const KMemoryRegion &slab_region = KMemoryLayout::GetSlabRegion();
KVirtualAddress address = slab_region.GetAddress();
/* Initialize slab type array to be in sorted order. */
KSlabType slab_types[KSlabType_Count];
@ -202,13 +203,21 @@ namespace ams::kern::init {
}
}
/* Track the gaps, so that we can free them to the unused slab tree. */
KVirtualAddress gap_start = address;
size_t gap_size = 0;
for (size_t i = 0; i < util::size(slab_types); i++) {
/* Add the random gap to the address. */
address += (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1];
const auto cur_gap = (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1];
address += cur_gap;
gap_size += cur_gap;
#define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...) \
case KSlabType_##NAME: \
address = InitializeSlabHeap<NAME>(address, COUNT); \
#define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...) \
case KSlabType_##NAME: \
if (COUNT > 0) { \
address = InitializeSlabHeap<NAME>(address, COUNT); \
} \
break;
/* Initialize the slabheap. */
@ -218,7 +227,17 @@ namespace ams::kern::init {
/* If we somehow get an invalid type, abort. */
MESOSPHERE_UNREACHABLE_DEFAULT_CASE();
}
/* If we've hit the end of a gap, free it. */
if (gap_start + gap_size != address) {
FreeUnusedSlabMemory(gap_start, gap_size);
gap_start = address;
gap_size = 0;
}
}
/* Free the end of the slab region. */
FreeUnusedSlabMemory(gap_start, gap_size + (slab_region.GetEndAddress() - GetInteger(address)));
}
}

6
libraries/libmesosphere/source/kern_initial_process.cpp
View file

@ -128,13 +128,13 @@ namespace ams::kern {
KProcess *new_process = nullptr;
{
/* Make page groups to represent the data. */
KPageGroup pg(std::addressof(Kernel::GetBlockInfoManager()));
KPageGroup workaround_pg(std::addressof(Kernel::GetBlockInfoManager()));
KPageGroup pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
KPageGroup workaround_pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
/* Populate the page group to represent the data. */
{
/* Allocate the previously unreserved pages. */
KPageGroup unreserve_pg(std::addressof(Kernel::GetBlockInfoManager()));
KPageGroup unreserve_pg(std::addressof(Kernel::GetSystemBlockInfoManager()));
MESOSPHERE_R_ABORT_UNLESS(Kernel::GetMemoryManager().AllocateAndOpen(std::addressof(unreserve_pg), unreserved_size / PageSize, KMemoryManager::EncodeOption(dst_pool, KMemoryManager::Direction_FromFront)));
/* Add the previously reserved pages. */

86
libraries/libmesosphere/source/kern_k_client_port.cpp
View file

@ -62,11 +62,46 @@ namespace ams::kern {
Result KClientPort::CreateSession(KClientSession **out) {
MESOSPHERE_ASSERT_THIS();
/* Declare the session we're going to allocate. */
KSession *session;
/* Reserve a new session from the resource limit. */
KScopedResourceReservation session_reservation(GetCurrentProcessPointer(), ams::svc::LimitableResource_SessionCountMax);
R_UNLESS(session_reservation.Succeeded(), svc::ResultLimitReached());
if (session_reservation.Succeeded()) {
/* Allocate a session normally. */
session = KSession::Create();
} else {
/* We couldn't reserve a session. Check that we support dynamically expanding the resource limit. */
R_UNLESS(GetCurrentProcess().GetResourceLimit() == std::addressof(Kernel::GetSystemResourceLimit()), svc::ResultLimitReached());
R_UNLESS(KTargetSystem::IsDynamicResourceLimitsEnabled(), svc::ResultLimitReached());
/* Try to allocate a session from unused slab memory. */
session = KSession::CreateFromUnusedSlabMemory();
R_UNLESS(session != nullptr, svc::ResultLimitReached());
/* Ensure that if we fail to allocate our session requests, we close the session we created. */
auto session_guard = SCOPE_GUARD { session->Close(); };
{
/* We want to add two KSessionRequests to the heap, to prevent request exhaustion. */
for (size_t i = 0; i < 2; ++i) {
KSessionRequest *request = KSessionRequest::CreateFromUnusedSlabMemory();
R_UNLESS(request != nullptr, svc::ResultLimitReached());
request->Close();
}
}
session_guard.Cancel();
/* We successfully allocated a session, so add the object we allocated to the resource limit. */
Kernel::GetSystemResourceLimit().Add(ams::svc::LimitableResource_SessionCountMax, 1);
}
/* Check that we successfully created a session. */
R_UNLESS(session != nullptr, svc::ResultOutOfResource());
/* Update the session counts. */
auto count_guard = SCOPE_GUARD { session->Close(); };
{
/* Atomically increment the number of sessions. */
s32 new_sessions;
@ -90,18 +125,7 @@ namespace ams::kern {
} while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed));
}
}
/* Create a new session. */
KSession *session = KSession::Create();
if (session == nullptr) {
/* Decrement the session count. */
const auto prev = m_num_sessions--;
if (prev == m_max_sessions) {
this->NotifyAvailable();
}
return svc::ResultOutOfResource();
}
count_guard.Cancel();
/* Initialize the session. */
session->Initialize(this, m_parent->GetName());
@ -128,11 +152,32 @@ namespace ams::kern {
Result KClientPort::CreateLightSession(KLightClientSession **out) {
MESOSPHERE_ASSERT_THIS();
/* Declare the session we're going to allocate. */
KLightSession *session;
/* Reserve a new session from the resource limit. */
KScopedResourceReservation session_reservation(GetCurrentProcessPointer(), ams::svc::LimitableResource_SessionCountMax);
R_UNLESS(session_reservation.Succeeded(), svc::ResultLimitReached());
if (session_reservation.Succeeded()) {
/* Allocate a session normally. */
session = KLightSession::Create();
} else {
/* We couldn't reserve a session. Check that we support dynamically expanding the resource limit. */
R_UNLESS(GetCurrentProcess().GetResourceLimit() == std::addressof(Kernel::GetSystemResourceLimit()), svc::ResultLimitReached());
R_UNLESS(KTargetSystem::IsDynamicResourceLimitsEnabled(), svc::ResultLimitReached());
/* Try to allocate a session from unused slab memory. */
session = KLightSession::CreateFromUnusedSlabMemory();
R_UNLESS(session != nullptr, svc::ResultLimitReached());
/* We successfully allocated a session, so add the object we allocated to the resource limit. */
Kernel::GetSystemResourceLimit().Add(ams::svc::LimitableResource_SessionCountMax, 1);
}
/* Check that we successfully created a session. */
R_UNLESS(session != nullptr, svc::ResultOutOfResource());
/* Update the session counts. */
auto count_guard = SCOPE_GUARD { session->Close(); };
{
/* Atomically increment the number of sessions. */
s32 new_sessions;
@ -156,18 +201,7 @@ namespace ams::kern {
} while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed));
}
}
/* Create a new session. */
KLightSession *session = KLightSession::Create();
if (session == nullptr) {
/* Decrement the session count. */
const auto prev = m_num_sessions--;
if (prev == m_max_sessions) {
this->NotifyAvailable();
}
return svc::ResultOutOfResource();
}
count_guard.Cancel();
/* Initialize the session. */
session->Initialize(this, m_parent->GetName());

4
libraries/libmesosphere/source/kern_k_dump_object.cpp
View file

@ -369,14 +369,14 @@ namespace ams::kern::KDumpObject {
/* KBlockInfo slab. */
{
MESOSPHERE_RELEASE_LOG("KBlockInfo\n");
auto &manager = Kernel::GetBlockInfoManager();
auto &manager = Kernel::GetSystemBlockInfoManager();
MESOSPHERE_RELEASE_LOG(" Cur=%6zu Peak=%6zu Max=%6zu\n", manager.GetUsed(), manager.GetPeak(), manager.GetCount());
}
/* Page Table slab. */
{
MESOSPHERE_RELEASE_LOG("Page Table\n");
auto &manager = Kernel::GetPageTableManager();
auto &manager = Kernel::GetSystemPageTableManager();
MESOSPHERE_RELEASE_LOG(" Cur=%6zu Peak=%6zu Max=%6zu\n", manager.GetUsed(), manager.GetPeak(), manager.GetCount());
}
}

2
libraries/libmesosphere/source/kern_k_event.cpp
View file

@ -37,7 +37,7 @@ namespace ams::kern {
void KEvent::Finalize() {
MESOSPHERE_ASSERT_THIS();
KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList>::Finalize();
KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList, true>::Finalize();
}
Result KEvent::Signal() {

1
libraries/libmesosphere/source/kern_k_memory_manager.cpp
View file

@ -37,6 +37,7 @@ namespace ams::kern {
void KMemoryManager::Initialize(KVirtualAddress management_region, size_t management_region_size) {
/* Clear the management region to zero. */
const KVirtualAddress management_region_end = management_region + management_region_size;
std::memset(GetVoidPointer(management_region), 0, management_region_size);

2
libraries/libmesosphere/source/kern_k_page_table_base.cpp
View file

@ -106,7 +106,7 @@ namespace ams::kern {
m_mapped_ipc_server_memory = 0;
m_memory_block_slab_manager = std::addressof(Kernel::GetSystemMemoryBlockManager());
m_block_info_manager = std::addressof(Kernel::GetBlockInfoManager());
m_block_info_manager = std::addressof(Kernel::GetSystemBlockInfoManager());
m_resource_limit = std::addressof(Kernel::GetSystemResourceLimit());
m_allocate_option = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);

23
libraries/libmesosphere/source/kern_k_process.cpp
View file

@ -260,8 +260,8 @@ namespace ams::kern {
const bool enable_das_merge = (params.flags & ams::svc::CreateProcessFlag_DisableDeviceAddressSpaceMerge) == 0;
const bool is_app = (params.flags & ams::svc::CreateProcessFlag_IsApplication) != 0;
auto *mem_block_manager = std::addressof(is_app ? Kernel::GetApplicationMemoryBlockManager() : Kernel::GetSystemMemoryBlockManager());
auto *block_info_manager = std::addressof(Kernel::GetBlockInfoManager());
auto *pt_manager = std::addressof(Kernel::GetPageTableManager());
auto *block_info_manager = std::addressof(is_app ? Kernel::GetApplicationBlockInfoManager() : Kernel::GetSystemBlockInfoManager());
auto *pt_manager = std::addressof(is_app ? Kernel::GetApplicationPageTableManager() : Kernel::GetSystemPageTableManager());
R_TRY(m_page_table.Initialize(m_process_id, as_type, enable_aslr, enable_das_merge, !enable_aslr, pool, params.code_address, params.code_num_pages * PageSize, mem_block_manager, block_info_manager, pt_manager, res_limit));
}
auto pt_guard = SCOPE_GUARD { m_page_table.Finalize(); };
@ -326,12 +326,17 @@ namespace ams::kern {
MESOSPHERE_ASSERT(m_system_resource_address != Null<KVirtualAddress>);
m_system_resource_num_pages = system_resource_num_pages;
/* Initialize managers. */
const size_t rc_size = util::AlignUp(KPageTableManager::CalculateReferenceCountSize(system_resource_size), PageSize);
/* Initialize slab heaps. */
const size_t rc_size = util::AlignUp(KPageTableSlabHeap::CalculateReferenceCountSize(system_resource_size), PageSize);
m_dynamic_page_manager.Initialize(m_system_resource_address + rc_size, system_resource_size - rc_size);
m_page_table_manager.Initialize(std::addressof(m_dynamic_page_manager), GetPointer<KPageTableManager::RefCount>(m_system_resource_address));
m_memory_block_slab_manager.Initialize(std::addressof(m_dynamic_page_manager));
m_block_info_manager.Initialize(std::addressof(m_dynamic_page_manager));
m_page_table_heap.Initialize(std::addressof(m_dynamic_page_manager), 0, GetPointer<KPageTableManager::RefCount>(m_system_resource_address));
m_memory_block_heap.Initialize(std::addressof(m_dynamic_page_manager), 0);
m_block_info_heap.Initialize(std::addressof(m_dynamic_page_manager), 0);
/* Initialize managers. */
m_page_table_manager.Initialize(std::addressof(m_dynamic_page_manager), std::addressof(m_page_table_heap));
m_memory_block_slab_manager.Initialize(std::addressof(m_dynamic_page_manager), std::addressof(m_memory_block_heap));
m_block_info_manager.Initialize(std::addressof(m_dynamic_page_manager), std::addressof(m_block_info_heap));
mem_block_manager = std::addressof(m_memory_block_slab_manager);
block_info_manager = std::addressof(m_block_info_manager);
@ -339,8 +344,8 @@ namespace ams::kern {
} else {
const bool is_app = (params.flags & ams::svc::CreateProcessFlag_IsApplication);
mem_block_manager = std::addressof(is_app ? Kernel::GetApplicationMemoryBlockManager() : Kernel::GetSystemMemoryBlockManager());
block_info_manager = std::addressof(Kernel::GetBlockInfoManager());
pt_manager = std::addressof(Kernel::GetPageTableManager());
block_info_manager = std::addressof(is_app ? Kernel::GetApplicationBlockInfoManager() : Kernel::GetSystemBlockInfoManager());
pt_manager = std::addressof(is_app ? Kernel::GetApplicationPageTableManager() : Kernel::GetSystemPageTableManager());
}
/* Ensure we don't leak any secure memory we allocated. */

43
libraries/libmesosphere/source/kern_k_resource_limit.cpp
View file

@ -49,7 +49,8 @@ namespace ams::kern {
KScopedLightLock lk(m_lock);
value = m_limit_values[which];
MESOSPHERE_ASSERT(value >= 0);
MESOSPHERE_ASSERT(m_current_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_values[which] <= m_peak_values[which]);
MESOSPHERE_ASSERT(m_peak_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_hints[which] <= m_current_values[which]);
}
@ -64,7 +65,8 @@ namespace ams::kern {
KScopedLightLock lk(m_lock);
value = m_current_values[which];
MESOSPHERE_ASSERT(value >= 0);
MESOSPHERE_ASSERT(m_current_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_values[which] <= m_peak_values[which]);
MESOSPHERE_ASSERT(m_peak_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_hints[which] <= m_current_values[which]);
}
@ -79,7 +81,8 @@ namespace ams::kern {
KScopedLightLock lk(m_lock);
value = m_peak_values[which];
MESOSPHERE_ASSERT(value >= 0);
MESOSPHERE_ASSERT(m_current_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_values[which] <= m_peak_values[which]);
MESOSPHERE_ASSERT(m_peak_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_hints[which] <= m_current_values[which]);
}
@ -93,7 +96,8 @@ namespace ams::kern {
{
KScopedLightLock lk(m_lock);
MESOSPHERE_ASSERT(m_current_values[which] >= 0);
MESOSPHERE_ASSERT(m_current_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_values[which] <= m_peak_values[which]);
MESOSPHERE_ASSERT(m_peak_values[which] <= m_limit_values[which]);
MESOSPHERE_ASSERT(m_current_hints[which] <= m_current_values[which]);
value = m_limit_values[which] - m_current_values[which];
}
@ -113,6 +117,37 @@ namespace ams::kern {
return ResultSuccess();
}
void KResourceLimit::Add(ams::svc::LimitableResource which, s64 value) {
MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KTargetSystem::IsDynamicResourceLimitsEnabled());
KScopedLightLock lk(m_lock);
/* Check that this is a true increase. */
MESOSPHERE_ABORT_UNLESS(value > 0);
/* Check that we can perform an increase. */
MESOSPHERE_ABORT_UNLESS(m_current_values[which] <= m_peak_values[which]);
MESOSPHERE_ABORT_UNLESS(m_peak_values[which] <= m_limit_values[which]);
MESOSPHERE_ABORT_UNLESS(m_current_hints[which] <= m_current_values[which]);
/* Check that the increase doesn't cause an overflow. */
const auto increased_limit = m_limit_values[which] + value;
const auto increased_current = m_current_values[which] + value;
const auto increased_hint = m_current_hints[which] + value;
MESOSPHERE_ABORT_UNLESS(m_limit_values[which] < increased_limit);
MESOSPHERE_ABORT_UNLESS(m_current_values[which] < increased_current);
MESOSPHERE_ABORT_UNLESS(m_current_hints[which] < increased_hint);
/* Add the value. */
m_limit_values[which] = increased_limit;
m_current_values[which] = increased_current;
m_current_hints[which] = increased_hint;
/* Update our peak. */
m_peak_values[which] = std::max(m_peak_values[which], increased_current);
}
bool KResourceLimit::Reserve(ams::svc::LimitableResource which, s64 value) {
return this->Reserve(which, value, KHardwareTimer::GetTick() + DefaultTimeout);
}

159
libraries/libmesosphere/source/kern_k_unused_slab_memory.cpp Normal file
View file

@ -0,0 +1,159 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern {
namespace {
class KUnusedSlabMemory : public util::IntrusiveRedBlackTreeBaseNode<KUnusedSlabMemory> {
NON_COPYABLE(KUnusedSlabMemory);
NON_MOVEABLE(KUnusedSlabMemory);
private:
size_t m_size;
public:
struct RedBlackKeyType {
size_t m_size;
constexpr ALWAYS_INLINE size_t GetSize() const {
return m_size;
}
};
template<typename T> requires (std::same_as<T, KUnusedSlabMemory> || std::same_as<T, RedBlackKeyType>)
static constexpr ALWAYS_INLINE int Compare(const T &lhs, const KUnusedSlabMemory &rhs) {
if (lhs.GetSize() < rhs.GetSize()) {
return -1;
} else {
return 1;
}
}
public:
constexpr KUnusedSlabMemory(size_t size) : m_size(size) { /* ... */ }
constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return reinterpret_cast<uintptr_t>(this); }
constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
};
static_assert(std::is_trivially_destructible<KUnusedSlabMemory>::value);
using KUnusedSlabMemoryTree = util::IntrusiveRedBlackTreeBaseTraits<KUnusedSlabMemory>::TreeType<KUnusedSlabMemory>;
constinit KLightLock g_unused_slab_memory_lock;
constinit KUnusedSlabMemoryTree g_unused_slab_memory_tree;
}
KVirtualAddress AllocateUnusedSlabMemory(size_t size, size_t alignment) {
/* Acquire exclusive access to the memory tree. */
KScopedLightLock lk(g_unused_slab_memory_lock);
/* Adjust size and alignment. */
size = std::max(size, sizeof(KUnusedSlabMemory));
alignment = std::max(alignment, alignof(KUnusedSlabMemory));
/* Find the smallest block which fits our allocation. */
KUnusedSlabMemory *best_fit = std::addressof(*g_unused_slab_memory_tree.nfind_key({ size - 1 }));
/* Ensure that the chunk is valid. */
size_t prefix_waste;
KVirtualAddress alloc_start;
KVirtualAddress alloc_last;
KVirtualAddress alloc_end;
KVirtualAddress chunk_last;
KVirtualAddress chunk_end;
while (true) {
/* Check that we still have a chunk satisfying our size requirement. */
if (AMS_UNLIKELY(best_fit == nullptr)) {
return Null<KVirtualAddress>;
}
/* Determine where the actual allocation would start. */
alloc_start = util::AlignUp(GetInteger(best_fit->GetAddress()), alignment);
if (AMS_LIKELY(alloc_start >= best_fit->GetAddress())) {
prefix_waste = alloc_start - best_fit->GetAddress();
alloc_end = alloc_start + size;
alloc_last = alloc_end - 1;
/* Check that the allocation remains in bounds. */
if (alloc_start <= alloc_last) {
chunk_end = best_fit->GetAddress() + best_fit->GetSize();
chunk_last = chunk_end - 1;
if (AMS_LIKELY(alloc_last <= chunk_last)) {
break;
}
}
}
/* Check the next smallest block. */
best_fit = best_fit->GetNext();
}
/* Remove the chunk we selected from the tree. */
g_unused_slab_memory_tree.erase(g_unused_slab_memory_tree.iterator_to(*best_fit));
std::destroy_at(best_fit);
/* If there's enough prefix waste due to alignment for a new chunk, insert it into the tree. */
if (prefix_waste >= sizeof(KUnusedSlabMemory)) {
std::construct_at(best_fit, prefix_waste);
g_unused_slab_memory_tree.insert(*best_fit);
}
/* If there's enough suffix waste after the allocation for a new chunk, insert it into the tree. */
if (alloc_last < alloc_end + sizeof(KUnusedSlabMemory) - 1 && alloc_end + sizeof(KUnusedSlabMemory) - 1 <= chunk_last) {
KUnusedSlabMemory *suffix_chunk = GetPointer<KUnusedSlabMemory>(alloc_end);
std::construct_at(suffix_chunk, chunk_end - alloc_end);
g_unused_slab_memory_tree.insert(*suffix_chunk);
}
/* Return the allocated memory. */
return alloc_start;
}
void FreeUnusedSlabMemory(KVirtualAddress address, size_t size) {
/* NOTE: This is called only during initialization, so we don't need exclusive access. */
/* Nintendo doesn't acquire the lock here, either. */
/* Check that there's anything at all for us to free. */
if (AMS_UNLIKELY(size == 0)) {
return;
}
/* Determine the start of the block. */
const KVirtualAddress block_start = util::AlignUp(GetInteger(address), alignof(KUnusedSlabMemory));
/* Check that there's space for a KUnusedSlabMemory to exist. */
if (AMS_UNLIKELY(std::numeric_limits<uintptr_t>::max() - sizeof(KUnusedSlabMemory) < GetInteger(block_start))) {
return;
}
/* Determine the end of the block region. */
const KVirtualAddress block_end = util::AlignDown(GetInteger(address) + size, alignof(KUnusedSlabMemory));
/* Check that the block remains within bounds. */
if (AMS_UNLIKELY(block_start + sizeof(KUnusedSlabMemory) - 1 > block_end - 1)){
return;
}
/* Create the block. */
KUnusedSlabMemory *block = GetPointer<KUnusedSlabMemory>(block_start);
std::construct_at(block, GetInteger(block_end) - GetInteger(block_start));
/* Insert the block into the tree. */
g_unused_slab_memory_tree.insert(*block);
}
}

33
libraries/libmesosphere/source/kern_kernel.cpp
View file

@ -66,15 +66,11 @@ namespace ams::kern {
void Kernel::InitializeResourceManagers(KVirtualAddress address, size_t size) {
/* Ensure that the buffer is suitable for our use. */
//const size_t app_size = ApplicationMemoryBlockSlabHeapSize * sizeof(KMemoryBlock);
//const size_t sys_size = SystemMemoryBlockSlabHeapSize * sizeof(KMemoryBlock);
//const size_t info_size = BlockInfoSlabHeapSize * sizeof(KBlockInfo);
//const size_t fixed_size = util::AlignUp(app_size + sys_size + info_size, PageSize);
MESOSPHERE_ABORT_UNLESS(util::IsAligned(GetInteger(address), PageSize));
MESOSPHERE_ABORT_UNLESS(util::IsAligned(size, PageSize));
/* Ensure that we have space for our reference counts. */
const size_t rc_size = util::AlignUp(KPageTableManager::CalculateReferenceCountSize(size), PageSize);
const size_t rc_size = util::AlignUp(KPageTableSlabHeap::CalculateReferenceCountSize(size), PageSize);
MESOSPHERE_ABORT_UNLESS(rc_size < size);
size -= rc_size;
@ -82,13 +78,28 @@ namespace ams::kern {
g_resource_manager_page_manager.Initialize(address, size);
/* Initialize the fixed-size slabheaps. */
s_app_memory_block_manager.Initialize(std::addressof(g_resource_manager_page_manager), ApplicationMemoryBlockSlabHeapSize);
s_sys_memory_block_manager.Initialize(std::addressof(g_resource_manager_page_manager), SystemMemoryBlockSlabHeapSize);
s_block_info_manager.Initialize(std::addressof(g_resource_manager_page_manager), BlockInfoSlabHeapSize);
s_app_memory_block_heap.Initialize(std::addressof(g_resource_manager_page_manager), ApplicationMemoryBlockSlabHeapSize);
s_sys_memory_block_heap.Initialize(std::addressof(g_resource_manager_page_manager), SystemMemoryBlockSlabHeapSize);
s_block_info_heap.Initialize(std::addressof(g_resource_manager_page_manager), BlockInfoSlabHeapSize);
/* Reserve all remaining pages for the page table manager. */
const size_t num_pt_pages = g_resource_manager_page_manager.GetCount() - g_resource_manager_page_manager.GetUsed();
s_page_table_manager.Initialize(std::addressof(g_resource_manager_page_manager), num_pt_pages, GetPointer<KPageTableManager::RefCount>(address + size));
/* Reserve all but a fixed number of remaining pages for the page table heap. */
const size_t num_pt_pages = g_resource_manager_page_manager.GetCount() - g_resource_manager_page_manager.GetUsed() - ReservedDynamicPageCount;
s_page_table_heap.Initialize(std::addressof(g_resource_manager_page_manager), num_pt_pages, GetPointer<KPageTableManager::RefCount>(address + size));
/* Setup the slab managers. */
KDynamicPageManager * const app_dynamic_page_manager = nullptr;
KDynamicPageManager * const sys_dynamic_page_manager = KTargetSystem::IsDynamicResourceLimitsEnabled() ? std::addressof(g_resource_manager_page_manager) : nullptr;
s_app_memory_block_manager.Initialize(app_dynamic_page_manager, std::addressof(s_app_memory_block_heap));
s_sys_memory_block_manager.Initialize(sys_dynamic_page_manager, std::addressof(s_sys_memory_block_heap));
s_app_block_info_manager.Initialize(app_dynamic_page_manager, std::addressof(s_block_info_heap));
s_sys_block_info_manager.Initialize(sys_dynamic_page_manager, std::addressof(s_block_info_heap));
s_app_page_table_manager.Initialize(app_dynamic_page_manager, std::addressof(s_page_table_heap));
s_sys_page_table_manager.Initialize(sys_dynamic_page_manager, std::addressof(s_page_table_heap));
/* Check that we have the correct number of dynamic pages available. */
MESOSPHERE_ABORT_UNLESS(g_resource_manager_page_manager.GetCount() - g_resource_manager_page_manager.GetUsed() == ReservedDynamicPageCount);
}
void Kernel::PrintLayout() {

22
libraries/libmesosphere/source/svc/kern_svc_event.cpp
View file

@ -60,12 +60,28 @@ namespace ams::kern::svc {
auto &process = GetCurrentProcess();
auto &handle_table = process.GetHandleTable();
/* Declare the event we're going to allocate. */
KEvent *event;
/* Reserve a new event from the process resource limit. */
KScopedResourceReservation event_reservation(std::addressof(process), ams::svc::LimitableResource_EventCountMax);
R_UNLESS(event_reservation.Succeeded(), svc::ResultLimitReached());
if (event_reservation.Succeeded()) {
/* Allocate an event normally. */
event = KEvent::Create();
} else {
/* We couldn't reserve an event. Check that we support dynamically expanding the resource limit. */
R_UNLESS(process.GetResourceLimit() == std::addressof(Kernel::GetSystemResourceLimit()), svc::ResultLimitReached());
R_UNLESS(KTargetSystem::IsDynamicResourceLimitsEnabled(), svc::ResultLimitReached());
/* Create a new event. */
KEvent *event = KEvent::Create();
/* Try to allocate an event from unused slab memory. */
event = KEvent::CreateFromUnusedSlabMemory();
R_UNLESS(event != nullptr, svc::ResultLimitReached());
/* We successfully allocated an event, so add the object we allocated to the resource limit. */
Kernel::GetSystemResourceLimit().Add(ams::svc::LimitableResource_EventCountMax, 1);
}
/* Check that we successfully created an event. */
R_UNLESS(event != nullptr, svc::ResultOutOfResource());
/* Initialize the event. */

38
libraries/libmesosphere/source/svc/kern_svc_session.cpp
View file

@ -27,12 +27,44 @@ namespace ams::kern::svc {
auto &process = GetCurrentProcess();
auto &handle_table = process.GetHandleTable();
/* Declare the session we're going to allocate. */
T *session;
/* Reserve a new session from the process resource limit. */
KScopedResourceReservation session_reservation(std::addressof(process), ams::svc::LimitableResource_SessionCountMax);
R_UNLESS(session_reservation.Succeeded(), svc::ResultLimitReached());
if (session_reservation.Succeeded()) {
/* Allocate a session normally. */
session = T::Create();
} else {
/* We couldn't reserve a session. Check that we support dynamically expanding the resource limit. */
R_UNLESS(process.GetResourceLimit() == std::addressof(Kernel::GetSystemResourceLimit()), svc::ResultLimitReached());
R_UNLESS(KTargetSystem::IsDynamicResourceLimitsEnabled(), svc::ResultLimitReached());
/* Create a new session. */
T *session = T::Create();
/* Try to allocate a session from unused slab memory. */
session = T::CreateFromUnusedSlabMemory();
R_UNLESS(session != nullptr, svc::ResultLimitReached());
/* If we're creating a KSession, we want to add two KSessionRequests to the heap, to prevent request exhaustion. */
/* NOTE: Nintendo checks if session->DynamicCast<KSession *>() != nullptr, but there's no reason to not do this statically. */
if constexpr (std::same_as<T, KSession>) {
/* Ensure that if we fail to allocate our session requests, we close the session we created. */
auto session_guard = SCOPE_GUARD { session->Close(); };
{
for (size_t i = 0; i < 2; ++i) {
KSessionRequest *request = KSessionRequest::CreateFromUnusedSlabMemory();
R_UNLESS(request != nullptr, svc::ResultLimitReached());
request->Close();
}
}
session_guard.Cancel();
}
/* We successfully allocated a session, so add the object we allocated to the resource limit. */
Kernel::GetSystemResourceLimit().Add(ams::svc::LimitableResource_SessionCountMax, 1);
}
/* Check that we successfully created a session. */
R_UNLESS(session != nullptr, svc::ResultOutOfResource());
/* Initialize the session. */

15
mesosphere/kernel/source/kern_kernel_instantiations.cpp
View file

@ -21,10 +21,6 @@ namespace ams::kern {
constinit Kernel::State Kernel::s_state = Kernel::State::Invalid;
constinit KResourceLimit Kernel::s_system_resource_limit;
KMemoryManager Kernel::s_memory_manager;
constinit KPageTableManager Kernel::s_page_table_manager;
constinit KMemoryBlockSlabManager Kernel::s_app_memory_block_manager;
constinit KMemoryBlockSlabManager Kernel::s_sys_memory_block_manager;
constinit KBlockInfoManager Kernel::s_block_info_manager;
constinit KSupervisorPageTable Kernel::s_supervisor_page_table;
constinit KUnsafeMemory Kernel::s_unsafe_memory;
constinit KWorkerTaskManager Kernel::s_worker_task_managers[KWorkerTaskManager::WorkerType_Count];
@ -33,6 +29,17 @@ namespace ams::kern {
constinit KInterruptTaskManager Kernel::s_interrupt_task_managers[cpu::NumCores];
constinit KHardwareTimer Kernel::s_hardware_timers[cpu::NumCores];
constinit KPageTableSlabHeap Kernel::s_page_table_heap;
constinit KMemoryBlockSlabHeap Kernel::s_app_memory_block_heap;
constinit KMemoryBlockSlabHeap Kernel::s_sys_memory_block_heap;
constinit KBlockInfoSlabHeap Kernel::s_block_info_heap;
constinit KPageTableManager Kernel::s_app_page_table_manager;
constinit KPageTableManager Kernel::s_sys_page_table_manager;
constinit KMemoryBlockSlabManager Kernel::s_app_memory_block_manager;
constinit KMemoryBlockSlabManager Kernel::s_sys_memory_block_manager;
constinit KBlockInfoManager Kernel::s_app_block_info_manager;
constinit KBlockInfoManager Kernel::s_sys_block_info_manager;
namespace {
constinit std::array<KThread, cpu::NumCores> g_main_threads;