diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp new file mode 100644 index 000000000..c52f28528 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::kern::arch::arm64 { + + template + concept SlabHeapNode = requires (T &t) { + { t.next } -> std::convertible_to; + }; + + template requires SlabHeapNode + ALWAYS_INLINE T *AllocateFromSlabAtomic(T **head) { + u32 tmp; + T *node, *next; + + __asm__ __volatile__( + "1:\n" + " ldaxr %[node], [%[head]]\n" + " cbz %[node], 2f\n" + " ldr %[next], [%[node]]\n" + " stlxr %w[tmp], %[next], [%[head]]\n" + " cbnz %w[tmp], 1b\n" + " b 3f\n" + "2:\n" + " clrex\n" + "3:\n" + : [tmp]"=&r"(tmp), [node]"=&r"(node), [next]"=&r"(next), [head]"+&r"(head) + : + : "cc", "memory" + ); + + return node; + } + + template requires SlabHeapNode + ALWAYS_INLINE void FreeToSlabAtomic(T **head, T *node) { + u32 tmp; + T *next; + + __asm__ __volatile__( + "1:\n" + " ldaxr %[next], [%[head]]\n" + " str %[next], [%[node]]\n" + " stlxr %w[tmp], %[node], [%[head]]\n" + " cbnz %w[tmp], 1b\n" + "2:\n" + : [tmp]"=&r"(tmp), [node]"+&r"(node), [next]"=&r"(next), [head]"+&r"(head) + : + : "cc", "memory" + ); + } + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp index 2d24df367..494777700 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp @@ -17,6 +17,21 @@ #include #include +#if defined(ATMOSPHERE_ARCH_ARM64) + + #include + namespace ams::kern { + using ams::kern::arch::arm64::AllocateFromSlabAtomic; + using ams::kern::arch::arm64::FreeToSlabAtomic; + } + +#else + + #error "Unknown architecture for KSlabHeapImpl" + +#endif + + namespace ams::kern { namespace impl { @@ -29,8 +44,8 @@ namespace ams::kern { Node *next; }; private: - std::atomic head; - size_t obj_size; + Node * head; + size_t obj_size; public: constexpr KSlabHeapImpl() : head(nullptr), obj_size(0) { MESOSPHERE_ASSERT_THIS(); } @@ -50,15 +65,7 @@ namespace ams::kern { void *Allocate() { MESOSPHERE_ASSERT_THIS(); - Node *ret = this->head.load(); - - do { - if (AMS_UNLIKELY(ret == nullptr)) { - break; - } - } while (!this->head.compare_exchange_weak(ret, ret->next)); - - return ret; + return AllocateFromSlabAtomic(std::addressof(this->head)); } void Free(void *obj) { @@ -66,10 +73,7 @@ namespace ams::kern { Node *node = reinterpret_cast(obj); - Node *cur_head = this->head.load(); - do { - node->next = cur_head; - } while (!this->head.compare_exchange_weak(cur_head, node)); + return FreeToSlabAtomic(std::addressof(this->head), node); } };