From c7fb3f720346b40cf3ec05d2bf3df2b6d34b1da2 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Wed, 11 Dec 2019 04:04:27 -0800 Subject: [PATCH] lmem: Implement Slab^H^H^H^HUnitHeap --- .../include/stratosphere/lmem.hpp | 1 + .../stratosphere/lmem/lmem_unit_heap.hpp | 46 +++ .../lmem/impl/lmem_impl_common_heap.hpp | 2 +- .../source/lmem/impl/lmem_impl_unit_heap.cpp | 263 ++++++++++++++++++ .../source/lmem/impl/lmem_impl_unit_heap.hpp | 38 +++ .../source/lmem/lmem_unit_heap.cpp | 81 ++++++ 6 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 libraries/libstratosphere/include/stratosphere/lmem/lmem_unit_heap.hpp create mode 100644 libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.cpp create mode 100644 libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.hpp create mode 100644 libraries/libstratosphere/source/lmem/lmem_unit_heap.cpp diff --git a/libraries/libstratosphere/include/stratosphere/lmem.hpp b/libraries/libstratosphere/include/stratosphere/lmem.hpp index 18b8f8d45..fa85facb6 100644 --- a/libraries/libstratosphere/include/stratosphere/lmem.hpp +++ b/libraries/libstratosphere/include/stratosphere/lmem.hpp @@ -17,3 +17,4 @@ #include "lmem/lmem_common.hpp" #include "lmem/lmem_exp_heap.hpp" +#include "lmem/lmem_unit_heap.hpp" diff --git a/libraries/libstratosphere/include/stratosphere/lmem/lmem_unit_heap.hpp b/libraries/libstratosphere/include/stratosphere/lmem/lmem_unit_heap.hpp new file mode 100644 index 000000000..565ad825b --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/lmem/lmem_unit_heap.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include "lmem_common.hpp" + +namespace ams::lmem { + + enum InfoPlacement { + InfoPlacement_Head, + InfoPlacement_Tail, + }; + + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, u32 option); + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, u32 option, s32 alignment, InfoPlacement info_placement); + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, u32 option, s32 alignment, HeapCommonHead *heap_head); + void DestroyUnitHeap(HeapHandle handle); + + void InvalidateUnitHeap(HeapHandle handle); + void ExtendUnitHeap(HeapHandle handle, size_t size); + + void *AllocateFromUnitHeap(HeapHandle handle); + void FreeToUnitHeap(HeapHandle handle, void *block); + + size_t GetUnitHeapUnitSize(HeapHandle handle); + s32 GetUnitHeapAlignment(HeapHandle handle); + size_t GetUnitHeapFreeCount(HeapHandle handle); + size_t GetUnitHeapUsedCount(HeapHandle handle); + + size_t GetUnitHeapRequiredSize(size_t unit_size, size_t unit_count, s32 alignment, bool internal_metadata); + +} diff --git a/libraries/libstratosphere/source/lmem/impl/lmem_impl_common_heap.hpp b/libraries/libstratosphere/source/lmem/impl/lmem_impl_common_heap.hpp index ba698be94..0e254f4fa 100644 --- a/libraries/libstratosphere/source/lmem/impl/lmem_impl_common_heap.hpp +++ b/libraries/libstratosphere/source/lmem/impl/lmem_impl_common_heap.hpp @@ -57,12 +57,12 @@ namespace ams::lmem::impl { return end - start; } - void InitializeHeapHead(HeapHead *out, u32 magic, void *start, void *end, u32 option); void FinalizeHeap(HeapHead *heap); bool ContainsAddress(HeapHandle handle, const void *address); size_t GetHeapTotalSize(HeapHandle handle); + /* Debug Fill */ u32 GetDebugFillValue(FillType type); u32 SetDebugFillValue(FillType type, u32 value); diff --git a/libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.cpp b/libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.cpp new file mode 100644 index 000000000..bbb59aa83 --- /dev/null +++ b/libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.cpp @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "lmem_impl_unit_heap.hpp" + +namespace ams::lmem::impl { + + namespace { + + constexpr size_t MinimumAlignment = 4; + + constexpr inline bool IsValidHeapHandle(HeapHandle handle) { + return handle->magic == UnitHeapMagic; + } + + constexpr inline UnitHeapHead *GetUnitHeapHead(HeapHead *heap_head) { + return &heap_head->impl_head.unit_heap_head; + } + + constexpr inline const UnitHeapHead *GetUnitHeapHead(const HeapHead *heap_head) { + return &heap_head->impl_head.unit_heap_head; + } + + inline UnitHead *PopUnit(UnitHeapList *list) { + if (UnitHead *block = list->head; block != nullptr) { + list->head = block->next; + return block; + } else { + return nullptr; + } + } + + inline void PushUnit(UnitHeapList *list, UnitHead *block) { + block->next = list->head; + list->head = block; + } + + } + + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, s32 alignment, u16 option, InfoPlacement info_placement, HeapCommonHead *heap_head) { + AMS_ASSERT(address != nullptr); + + /* Correct alignment, validate. */ + if (alignment == 1 || alignment == 2) { + alignment = 4; + } + AMS_ASSERT(util::IsAligned(alignment, MinimumAlignment)); + AMS_ASSERT(static_cast(MinimumAlignment) <= alignment); + AMS_ASSERT(unit_size >= sizeof(uintptr_t)); + + /* Setup heap metadata. */ + UnitHeapHead *unit_heap = nullptr; + void *heap_start = nullptr; + void *heap_end = nullptr; + if (heap_head == nullptr) { + /* Internal heap metadata. */ + if (info_placement == InfoPlacement_Head) { + heap_head = reinterpret_cast(util::AlignUp(address, MinimumAlignment)); + unit_heap = GetUnitHeapHead(heap_head); + heap_end = util::AlignDown(reinterpret_cast(reinterpret_cast(address) + size), MinimumAlignment); + heap_start = util::AlignUp(reinterpret_cast(reinterpret_cast(heap_head) + sizeof(HeapHead)), alignment); + } else if (info_placement == InfoPlacement_Tail) { + heap_end = util::AlignDown(reinterpret_cast(reinterpret_cast(address) + size - sizeof(HeapHead)), MinimumAlignment); + heap_head = reinterpret_cast(heap_end); + unit_heap = GetUnitHeapHead(heap_head); + heap_start = util::AlignUp(address, alignment); + } else { + AMS_ASSERT(false); + } + } else { + /* External heap metadata. */ + unit_heap = GetUnitHeapHead(heap_head); + heap_end = util::AlignDown(reinterpret_cast(reinterpret_cast(address) + size), MinimumAlignment); + heap_start = util::AlignUp(address, alignment); + } + + /* Correct unit size. */ + unit_size = util::AlignUp(unit_size, alignment); + + /* Don't allow a heap with start after end. */ + if (heap_start > heap_end) { + return nullptr; + } + + /* Don't allow a heap with no units. */ + size_t max_units = GetPointerDifference(heap_start, heap_end) / unit_size; + if (max_units == 0) { + return nullptr; + } + + /* Set real heap end. */ + heap_end = reinterpret_cast(reinterpret_cast(heap_start) + max_units * unit_size); + + /* Initialize the parent heap. */ + InitializeHeapHead(heap_head, UnitHeapMagic, heap_start, heap_end, option); + + /* Initialize the actual unit heap. */ + { + unit_heap->free_list.head = reinterpret_cast(heap_start); + unit_heap->unit_size = unit_size; + unit_heap->alignment = alignment; + unit_heap->num_units = 0; + + /* Create the new units. */ + UnitHead *cur_tail = unit_heap->free_list.head; + for (size_t i = 0; i < max_units - 1; i++) { + cur_tail->next = reinterpret_cast(reinterpret_cast(cur_tail) + unit_size); + cur_tail = cur_tail->next; + } + cur_tail->next = nullptr; + } + + /* Return the heap header as handle. */ + return heap_head; + } + + void DestroyUnitHeap(HeapHandle handle) { + AMS_ASSERT(IsValidHeapHandle(handle)); + + /* Validate that the heap has no living units. */ + UnitHeapHead *unit_heap = GetUnitHeapHead(handle); + if (unit_heap->free_list.head != nullptr) { + AMS_ASSERT(unit_heap->num_units == 0); + unit_heap->free_list.head = nullptr; + } + + FinalizeHeap(handle); + } + + void InvalidateUnitHeap(HeapHandle handle) { + AMS_ASSERT(IsValidHeapHandle(handle)); + GetUnitHeapHead(handle)->free_list.head = nullptr; + } + + void ExtendUnitHeap(HeapHandle handle, size_t size) { + AMS_ASSERT(IsValidHeapHandle(handle)); + + /* Find the current tail unit, and insert as next. */ + UnitHeapHead *unit_heap = GetUnitHeapHead(handle); + UnitHead *cur_tail; + if (unit_heap->free_list.head != nullptr) { + cur_tail = unit_heap->free_list.head; + + while (cur_tail->next != nullptr) { + cur_tail = cur_tail->next; + } + + cur_tail->next = reinterpret_cast(handle->heap_end); + cur_tail = cur_tail->next; + cur_tail->next = nullptr; + } else { + /* All units are allocated, so set the free list to be at the end of the heap area. */ + unit_heap->free_list.head = reinterpret_cast(handle->heap_end); + cur_tail = unit_heap->free_list.head; + cur_tail->next = nullptr; + } + + /* Calculate new unit extents. */ + void *new_units_start = handle->heap_end; + void *new_units_end = reinterpret_cast(reinterpret_cast(new_units_start) + size); + size_t num_new_units = GetPointerDifference(new_units_start, new_units_end) / unit_heap->unit_size; + AMS_ASSERT(num_new_units > 0); + + /* Create the new units. */ + for (size_t i = 0; i < num_new_units - 1; i++) { + cur_tail->next = reinterpret_cast(reinterpret_cast(cur_tail) + unit_heap->unit_size); + cur_tail = cur_tail->next; + } + cur_tail->next = nullptr; + + /* Note that the heap is bigger. */ + handle->heap_end = new_units_end; + } + + void *AllocateFromUnitHeap(HeapHandle handle) { + AMS_ASSERT(IsValidHeapHandle(handle)); + + /* Allocate a unit. */ + UnitHeapHead *unit_heap = GetUnitHeapHead(handle); + UnitHead *unit = PopUnit(&unit_heap->free_list); + if (unit != nullptr) { + /* Fill memory with pattern for debug, if needed. */ + FillAllocatedMemory(handle, unit, unit_heap->unit_size); + + /* Note that we allocated a unit. */ + unit_heap->num_units++; + } + + return unit; + } + + void FreeToUnitHeap(HeapHandle handle, void *block) { + AMS_ASSERT(IsValidHeapHandle(handle)); + + /* Allow Free(nullptr) to succeed. */ + if (block == nullptr) { + return; + } + + /* Fill memory with pattern for debug, if needed. */ + UnitHeapHead *unit_heap = GetUnitHeapHead(handle); + FillFreedMemory(handle, block, unit_heap->unit_size); + + /* Push the unit onto the free list. */ + PushUnit(&unit_heap->free_list, reinterpret_cast(block)); + + /* Note that we freed a unit. */ + unit_heap->num_units--; + } + + size_t GetUnitHeapUnitSize(HeapHandle handle) { + AMS_ASSERT(IsValidHeapHandle(handle)); + return GetUnitHeapHead(handle)->unit_size; + } + + s32 GetUnitHeapAlignment(HeapHandle handle) { + AMS_ASSERT(IsValidHeapHandle(handle)); + return GetUnitHeapHead(handle)->alignment; + } + + size_t GetUnitHeapFreeCount(HeapHandle handle) { + AMS_ASSERT(IsValidHeapHandle(handle)); + + size_t count = 0; + for (UnitHead *cur = GetUnitHeapHead(handle)->free_list.head; cur != nullptr; cur = cur->next) { + count++; + } + return count; + } + + size_t GetUnitHeapUsedCount(HeapHandle handle) { + AMS_ASSERT(IsValidHeapHandle(handle)); + return GetUnitHeapHead(handle)->num_units; + } + + size_t GetUnitHeapRequiredSize(size_t unit_size, size_t unit_count, s32 alignment, bool internal_metadata) { + /* Nintendo does not round up alignment here, even though they do so in CreateUnitHeap. */ + /* We will round up alignment to return more accurate results. */ + if (alignment == 1 || alignment == 2) { + alignment = 4; + } + + AMS_ASSERT(util::IsAligned(alignment, MinimumAlignment)); + AMS_ASSERT(static_cast(MinimumAlignment) <= alignment); + AMS_ASSERT(unit_size >= sizeof(uintptr_t)); + + return (alignment - 1) + util::AlignUp(unit_size, alignment) + (internal_metadata ? sizeof(HeapHead) : 0); + } + +} diff --git a/libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.hpp b/libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.hpp new file mode 100644 index 000000000..4c4dd5a97 --- /dev/null +++ b/libraries/libstratosphere/source/lmem/impl/lmem_impl_unit_heap.hpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include "lmem_impl_common_heap.hpp" + +namespace ams::lmem::impl { + + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, s32 alignment, u16 option, InfoPlacement info_placement, HeapCommonHead *heap_head); + void DestroyUnitHeap(HeapHandle handle); + + void InvalidateUnitHeap(HeapHandle handle); + void ExtendUnitHeap(HeapHandle handle, size_t size); + + void *AllocateFromUnitHeap(HeapHandle handle); + void FreeToUnitHeap(HeapHandle handle, void *block); + + size_t GetUnitHeapUnitSize(HeapHandle handle); + s32 GetUnitHeapAlignment(HeapHandle handle); + size_t GetUnitHeapFreeCount(HeapHandle handle); + size_t GetUnitHeapUsedCount(HeapHandle handle); + + size_t GetUnitHeapRequiredSize(size_t unit_size, size_t unit_count, s32 alignment, bool internal_metadata); + +} \ No newline at end of file diff --git a/libraries/libstratosphere/source/lmem/lmem_unit_heap.cpp b/libraries/libstratosphere/source/lmem/lmem_unit_heap.cpp new file mode 100644 index 000000000..3b613bb71 --- /dev/null +++ b/libraries/libstratosphere/source/lmem/lmem_unit_heap.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2018-2019 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "impl/lmem_impl_unit_heap.hpp" + +namespace ams::lmem { + + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, u32 option) { + return impl::CreateUnitHeap(address, size, unit_size, DefaultAlignment, static_cast(option), InfoPlacement_Head, nullptr); + } + + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, u32 option, s32 alignment, InfoPlacement info_placement) { + return impl::CreateUnitHeap(address, size, unit_size, alignment, static_cast(option), info_placement, nullptr); + } + + HeapHandle CreateUnitHeap(void *address, size_t size, size_t unit_size, u32 option, s32 alignment, HeapCommonHead *heap_head) { + return impl::CreateUnitHeap(address, size, unit_size, alignment, static_cast(option), InfoPlacement_Head, heap_head); + } + + void DestroyUnitHeap(HeapHandle handle) { + impl::DestroyUnitHeap(handle); + } + + void InvalidateUnitHeap(HeapHandle handle) { + impl::ScopedHeapLock lk(handle); + impl::InvalidateUnitHeap(handle); + } + + void ExtendUnitHeap(HeapHandle handle, size_t size) { + impl::ScopedHeapLock lk(handle); + impl::ExtendUnitHeap(handle, size); + } + + void *AllocateFromUnitHeap(HeapHandle handle) { + impl::ScopedHeapLock lk(handle); + return impl::AllocateFromUnitHeap(handle); + } + + void FreeToUnitHeap(HeapHandle handle, void *block) { + impl::ScopedHeapLock lk(handle); + impl::FreeToUnitHeap(handle, block); + } + + size_t GetUnitHeapUnitSize(HeapHandle handle) { + /* Nintendo doesn't acquire a lock here. */ + return impl::GetUnitHeapUnitSize(handle); + } + + s32 GetUnitHeapAlignment(HeapHandle handle) { + /* Nintendo doesn't acquire a lock here. */ + return impl::GetUnitHeapAlignment(handle); + } + + size_t GetUnitHeapFreeCount(HeapHandle handle) { + impl::ScopedHeapLock lk(handle); + return impl::GetUnitHeapFreeCount(handle); + } + + size_t GetUnitHeapUsedCount(HeapHandle handle) { + impl::ScopedHeapLock lk(handle); + return impl::GetUnitHeapUsedCount(handle); + } + + size_t GetUnitHeapRequiredSize(size_t unit_size, size_t unit_count, s32 alignment, bool internal_metadata) { + return impl::GetUnitHeapRequiredSize(unit_size, unit_count, alignment, internal_metadata); + } + +}