ncm: use static memory pools for different allocations

libraries/libstratosphere/include/stratosphere/kvdb/kvdb_auto_buffer.hpp

@@ -49,7 +49,7 @@ namespace ams::kvdb {
 
             void Reset() {
                 if (this->buffer != nullptr) {
-                    std::free(this->buffer);
+                    delete[] this->buffer;
                     this->buffer = nullptr;
                     this->size = 0;
                 }
@@ -68,7 +68,7 @@ namespace ams::kvdb {
                 AMS_ABORT_UNLESS(this->buffer == nullptr);
 
                 /* Allocate a buffer. */
-                this->buffer = static_cast<u8 *>(std::malloc(size));
+                this->buffer = new (std::nothrow) u8[size];
                 R_UNLESS(this->buffer != nullptr, ResultAllocationFailed());
 
                 this->size = size;

libraries/libstratosphere/include/stratosphere/kvdb/kvdb_memory_key_value_store.hpp

@@ -93,13 +93,14 @@ namespace ams::kvdb {
                     size_t count;
                     size_t capacity;
                     Entry *entries;
+                    MemoryResource *memory_resource;
                 public:
-                    Index() : count(0), capacity(0), entries(nullptr) { /* ... */ }
+                    Index() : count(0), capacity(0), entries(nullptr), memory_resource(nullptr) { /* ... */ }
 
                     ~Index() {
                         if (this->entries != nullptr) {
                             this->ResetEntries();
-                            std::free(this->entries);
+                            this->memory_resource->Deallocate(this->entries, sizeof(Entry) * this->capacity);
                             this->entries = nullptr;
                         }
                     }
@@ -114,30 +115,25 @@ namespace ams::kvdb {
 
                     void ResetEntries() {
                         for (size_t i = 0; i < this->count; i++) {
-                            std::free(this->entries[i].GetValuePointer());
+                            this->memory_resource->Deallocate(this->entries[i].GetValuePointer(), this->entries[i].GetValueSize());
                         }
                         this->count = 0;
                     }
 
-                    Result Initialize(size_t capacity) {
+                    Result Initialize(size_t capacity, MemoryResource *mr) {
-                        this->entries = reinterpret_cast<Entry *>(std::malloc(sizeof(Entry) * capacity));
+                        this->entries = reinterpret_cast<Entry *>(mr->Allocate(sizeof(Entry) * capacity));
                         R_UNLESS(this->entries != nullptr, ResultAllocationFailed());
                         this->capacity = capacity;
+                        this->memory_resource = mr;
                         return ResultSuccess();
                     }
 
                     Result Set(const Key &key, const void *value, size_t value_size) {
-                        /* Allocate new value. */
-                        void *new_value = std::malloc(value_size);
-                        R_UNLESS(new_value != nullptr, ResultAllocationFailed());
-                        auto value_guard = SCOPE_GUARD { std::free(new_value); };
-                        std::memcpy(new_value, value, value_size);
-
                         /* Find entry for key. */
                         Entry *it = this->lower_bound(key);
                         if (it != this->end() && it->GetKey() == key) {
                             /* Entry already exists. Free old value. */
-                            std::free(it->GetValuePointer());
+                            this->memory_resource->Deallocate(it->GetValuePointer(), it->GetValueSize());
                         } else {
                             /* We need to add a new entry. Check we have room, move future keys forward. */
                             R_UNLESS(this->count < this->capacity, ResultOutOfKeyResource());
@@ -145,8 +141,12 @@ namespace ams::kvdb {
                             this->count++;
                         }
 
+                        /* Allocate new value. */
+                        void *new_value = this->memory_resource->Allocate(value_size);
+                        R_UNLESS(new_value != nullptr, ResultAllocationFailed());
+                        std::memcpy(new_value, value, value_size);
+
                         /* Save the new Entry in the map. */
-                        value_guard.Cancel();
                         *it = Entry(key, new_value, value_size);
                         return ResultSuccess();
                     }
@@ -164,7 +164,7 @@ namespace ams::kvdb {
                         R_UNLESS(it != this->end(), ResultKeyNotFound());
 
                         /* Free the value, move entries back. */
-                        std::free(it->GetValuePointer());
+                        this->memory_resource->Deallocate(it->GetValuePointer(), it->GetValueSize());
                         std::memmove(it, it + 1, sizeof(*it) * (this->end() - (it + 1)));
                         this->count--;
                         return ResultSuccess();
@@ -258,10 +258,11 @@ namespace ams::kvdb {
             Index index;
             Path path;
             Path temp_path;
+            MemoryResource *memory_resource;
         public:
             MemoryKeyValueStore() { /* ... */ }
 
-            Result Initialize(const char *dir, size_t capacity) {
+            Result Initialize(const char *dir, size_t capacity, MemoryResource *mr) {
                 /* Ensure that the passed path is a directory. */
                 fs::DirectoryEntryType entry_type;
                 R_TRY(fs::GetEntryType(std::addressof(entry_type), dir));
@@ -272,18 +273,21 @@ namespace ams::kvdb {
                 this->temp_path.SetFormat("%s%s", dir, "/imkvdb.tmp");
 
                 /* Initialize our index. */
-                R_TRY(this->index.Initialize(capacity));
+                R_TRY(this->index.Initialize(capacity, mr));
+                this->memory_resource = mr;
+
                 return ResultSuccess();
             }
 
-            Result Initialize(size_t capacity) {
+            Result Initialize(size_t capacity, MemoryResource *mr) {
                 /* This initializes without an archive file. */
                 /* A store initialized this way cannot have its contents loaded from or flushed to disk. */
                 this->path.Set("");
                 this->temp_path.Set("");
 
                 /* Initialize our index. */
-                R_TRY(this->index.Initialize(capacity));
+                R_TRY(this->index.Initialize(capacity, mr));
+                this->memory_resource = mr;
                 return ResultSuccess();
             }
 
@@ -319,9 +323,9 @@ namespace ams::kvdb {
                         R_TRY(reader.GetEntrySize(&key_size, &value_size));
 
                         /* Allocate memory for value. */
-                        void *new_value = std::malloc(value_size);
+                        void *new_value = this->memory_resource->Allocate(value_size);
                         R_UNLESS(new_value != nullptr, ResultAllocationFailed());
-                        auto value_guard = SCOPE_GUARD { std::free(new_value); };
+                        auto value_guard = SCOPE_GUARD { this->memory_resource->Deallocate(new_value, value_size); };
 
                         /* Read key and value. */
                         Key key;

libraries/libstratosphere/include/stratosphere/ncm/ncm_auto_buffer.hpp

@@ -49,7 +49,7 @@ namespace ams::ncm {
 
             void Reset() {
                 if (this->buffer != nullptr) {
-                    std::free(this->buffer);
+                    delete[] this->buffer;
                     this->buffer = nullptr;
                     this->size = 0;
                 }
@@ -68,7 +68,7 @@ namespace ams::ncm {
                 AMS_ABORT_UNLESS(this->buffer == nullptr);
 
                 /* Allocate a buffer. */
-                this->buffer = static_cast<u8 *>(std::malloc(size));
+                this->buffer = new (std::nothrow) u8[size];
                 R_UNLESS(this->buffer != nullptr, ResultAllocationFailed());
 
                 this->size = size;
@@ -85,4 +85,5 @@ namespace ams::ncm {
                 return ResultSuccess();
             }
     };
+
 }

libraries/libstratosphere/include/stratosphere/ncm/ncm_content_manager_impl.hpp

@@ -30,6 +30,16 @@
 
 namespace ams::ncm {
 
+    class ContentMetaMemoryResource {
+        private:
+            mem::StandardAllocator allocator;
+            sf::StandardAllocatorMemoryResource memory_resource;
+        public:
+            ContentMetaMemoryResource(void *heap, size_t heap_size) : allocator(heap, heap_size), memory_resource(std::addressof(allocator)) { /* ... */ }
+
+            MemoryResource *Get() { return std::addressof(this->memory_resource); }
+    };
+
     struct SystemSaveDataInfo {
         u64 id;
         u64 size;
@@ -67,6 +77,7 @@ namespace ams::ncm {
                 SystemSaveDataInfo info;
                 std::shared_ptr<IContentMetaDatabase> content_meta_database;
                 std::optional<kvdb::MemoryKeyValueStore<ContentMetaKey>> kvs;
+                ContentMetaMemoryResource *memory_resource;
                 u32 max_content_metas;
 
                 ContentMetaDatabaseRoot() { /* ... */ }
@@ -92,8 +103,8 @@ namespace ams::ncm {
             Result InitializeContentStorageRoot(ContentStorageRoot *out, StorageId storage_id, fs::ContentStorageId content_storage_id);
             Result InitializeGameCardContentStorageRoot(ContentStorageRoot *out);
 
-            Result InitializeContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, StorageId storage_id, const SystemSaveDataInfo &info, size_t max_content_metas);
+            Result InitializeContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, StorageId storage_id, const SystemSaveDataInfo &info, size_t max_content_metas, ContentMetaMemoryResource *mr);
-            Result InitializeGameCardContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, size_t max_content_metas);
+            Result InitializeGameCardContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, size_t max_content_metas, ContentMetaMemoryResource *mr);
 
             Result BuildContentMetaDatabase(StorageId storage_id);
             Result ImportContentMetaDatabase(StorageId storage_id, bool from_signed_partition);

libraries/libstratosphere/include/stratosphere/sf.hpp

@@ -17,6 +17,7 @@
 #pragma once
 
 #include "sf/sf_common.hpp"
+#include "sf/sf_mem_utility.hpp"
 #include "sf/sf_service_object.hpp"
 #include "sf/hipc/sf_hipc_server_session_manager.hpp"
 

libraries/libstratosphere/include/stratosphere/sf/sf_common.hpp

@@ -16,6 +16,6 @@
 
 #pragma once
 #include <vapours.hpp>
-#include "../ams.hpp"
+#include <stratosphere/ams.hpp>
-#include "../os.hpp"
+#include <stratosphere/os.hpp>
-#include "../sm/sm_types.hpp"
+#include <stratosphere/sm/sm_types.hpp>

libraries/libstratosphere/include/stratosphere/sf/sf_mem_utility.hpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+#include <stratosphere/sf/sf_common.hpp>
+#include <stratosphere/mem.hpp>
+
+namespace ams::sf {
+
+    class StandardAllocatorMemoryResource : public MemoryResource {
+        private:
+            mem::StandardAllocator *standard_allocator;
+        public:
+            explicit StandardAllocatorMemoryResource(mem::StandardAllocator *sa) : standard_allocator(sa) { /* ... */ }
+
+            mem::StandardAllocator *GetAllocator() const { return this->standard_allocator; }
+        private:
+            virtual void *AllocateImpl(size_t size, size_t alignment) override {
+                return this->standard_allocator->Allocate(size, alignment);
+            }
+
+            virtual void DeallocateImpl(void *buffer, size_t size, size_t alignment) override {
+                return this->standard_allocator->Free(buffer);
+            }
+
+            virtual bool IsEqualImpl(const MemoryResource &resource) const {
+                return this == std::addressof(resource);
+            }
+    };
+
+}

libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_central.cpp

@@ -22,8 +22,6 @@ namespace ams::mem::impl::heap {
 
     namespace {
 
-        char g_log_buf[4_KB];
-
         void InitializeSpanPage(SpanPage *sp) {
             static_assert(SpanPage::MaxSpanCount <= BITSIZEOF(u64));
             constexpr size_t NumUnusedBits = BITSIZEOF(u64) - SpanPage::MaxSpanCount;

libraries/libstratosphere/source/ncm/ncm_content_management_utils.cpp

@@ -73,12 +73,11 @@ namespace ams::ncm {
 
         /* Allocate space to hold the converted meta. */
         const size_t meta_size = package_meta_reader.CalculateConvertContentMetaSize();
-        void *meta = std::malloc(meta_size);
+        std::unique_ptr<char[]> meta(new (std::nothrow) char[meta_size]);
-        ON_SCOPE_EXIT { std::free(meta); };
 
         /* Convert the meta from packaged form to normal form. */
-        package_meta_reader.ConvertToContentMeta(meta, meta_size, meta_info);
+        package_meta_reader.ConvertToContentMeta(meta.get(), meta_size, meta_info);
-        ncm::ContentMetaReader meta_reader(meta, meta_size);
+        ncm::ContentMetaReader meta_reader(meta.get(), meta_size);
 
         /* Insert the new metas into the database. */
         R_TRY(this->db->Set(package_meta_reader.GetKey(), meta_reader.GetData(), meta_reader.GetSize()));

libraries/libstratosphere/source/ncm/ncm_content_manager_impl.cpp

@@ -24,6 +24,14 @@ namespace ams::ncm {
 
     namespace {
 
+        alignas(os::MemoryPageSize) u8 g_system_content_meta_database_heap[512_KB];
+        alignas(os::MemoryPageSize) u8 g_gamecard_content_meta_database_heap[512_KB];
+        alignas(os::MemoryPageSize) u8 g_sd_and_user_content_meta_database_heap[2_MB + 512_KB];
+
+        ContentMetaMemoryResource g_system_content_meta_memory_resource(g_system_content_meta_database_heap, sizeof(g_system_content_meta_database_heap));
+        ContentMetaMemoryResource g_gamecard_content_meta_memory_resource(g_gamecard_content_meta_database_heap, sizeof(g_gamecard_content_meta_database_heap));
+        ContentMetaMemoryResource g_sd_and_user_content_meta_memory_resource(g_sd_and_user_content_meta_database_heap, sizeof(g_sd_and_user_content_meta_database_heap));
+
         constexpr fs::SystemSaveDataId BuiltInSystemSaveDataId = 0x8000000000000120;
         constexpr u64 BuiltInSystemSaveDataSize                = 0x6c000;
         constexpr u64 BuiltInSystemSaveDataJournalSize         = 0x6c000;
@@ -216,10 +224,11 @@ namespace ams::ncm {
         return ResultSuccess();
     }
 
-    Result ContentManagerImpl::InitializeContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, StorageId storage_id, const SystemSaveDataInfo &info, size_t max_content_metas) {
+    Result ContentManagerImpl::InitializeContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, StorageId storage_id, const SystemSaveDataInfo &info, size_t max_content_metas, ContentMetaMemoryResource *memory_resource) {
         out->storage_id            = storage_id;
         out->info                  = info;
         out->max_content_metas     = max_content_metas;
+        out->memory_resource       = memory_resource;
         out->content_meta_database = nullptr;
         out->kvs                   = std::nullopt;
 
@@ -231,9 +240,10 @@ namespace ams::ncm {
         return ResultSuccess();
     }
 
-    Result ContentManagerImpl::InitializeGameCardContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, size_t max_content_metas) {
+    Result ContentManagerImpl::InitializeGameCardContentMetaDatabaseRoot(ContentMetaDatabaseRoot *out, size_t max_content_metas, ContentMetaMemoryResource *memory_resource) {
         out->storage_id            = StorageId::GameCard;
         out->max_content_metas     = max_content_metas;
+        out->memory_resource       = memory_resource;
         out->content_meta_database = nullptr;
         out->kvs                   = std::nullopt;
 
@@ -332,7 +342,7 @@ namespace ams::ncm {
         R_TRY(this->ActivateContentStorage(StorageId::BuiltInSystem));
 
         /* Next, the BuiltInSystem content meta entry. */
-        R_TRY(this->InitializeContentMetaDatabaseRoot(&this->content_meta_database_roots[this->num_content_meta_entries++], StorageId::BuiltInSystem, BuiltInSystemSystemSaveDataInfo, MaxBuiltInSystemContentMetaCount));
+        R_TRY(this->InitializeContentMetaDatabaseRoot(&this->content_meta_database_roots[this->num_content_meta_entries++], StorageId::BuiltInSystem, BuiltInSystemSystemSaveDataInfo, MaxBuiltInSystemContentMetaCount, std::addressof(g_system_content_meta_memory_resource)));
 
         if (R_FAILED(this->VerifyContentMetaDatabase(StorageId::BuiltInSystem))) {
             R_TRY(this->CreateContentMetaDatabase(StorageId::BuiltInSystem));
@@ -360,18 +370,18 @@ namespace ams::ncm {
 
         /* Now for BuiltInUser's content storage and content meta entries. */
         R_TRY(this->InitializeContentStorageRoot(&this->content_storage_roots[this->num_content_storage_entries++], StorageId::BuiltInUser, fs::ContentStorageId::User));
-        R_TRY(this->InitializeContentMetaDatabaseRoot(&this->content_meta_database_roots[this->num_content_meta_entries++], StorageId::BuiltInUser, BuiltInUserSystemSaveDataInfo, MaxBuiltInUserContentMetaCount));
+        R_TRY(this->InitializeContentMetaDatabaseRoot(&this->content_meta_database_roots[this->num_content_meta_entries++], StorageId::BuiltInUser, BuiltInUserSystemSaveDataInfo, MaxBuiltInUserContentMetaCount, std::addressof(g_sd_and_user_content_meta_memory_resource)));
 
         /* Beyond this point, N uses hardcoded indices. */
 
         /* Next SdCard's content storage and content meta entries. */
         R_TRY(this->InitializeContentStorageRoot(&this->content_storage_roots[2], StorageId::SdCard, fs::ContentStorageId::SdCard));
-        R_TRY(this->InitializeContentMetaDatabaseRoot(&this->content_meta_database_roots[2], StorageId::SdCard, SdCardSystemSaveDataInfo, MaxSdCardContentMetaCount));
+        R_TRY(this->InitializeContentMetaDatabaseRoot(&this->content_meta_database_roots[2], StorageId::SdCard, SdCardSystemSaveDataInfo, MaxSdCardContentMetaCount, std::addressof(g_sd_and_user_content_meta_memory_resource)));
 
         /* GameCard's content storage and content meta entries. */
         /* N doesn't set a content storage id for game cards, so we'll just use 0 (System). */
         R_TRY(this->InitializeGameCardContentStorageRoot(&this->content_storage_roots[3]));
-        R_TRY(this->InitializeGameCardContentMetaDatabaseRoot(&this->content_meta_database_roots[3], MaxGameCardContentMetaCount));
+        R_TRY(this->InitializeGameCardContentMetaDatabaseRoot(&this->content_meta_database_roots[3], MaxGameCardContentMetaCount, std::addressof(g_gamecard_content_meta_memory_resource)));
 
         this->initialized = true;
         return ResultSuccess();
@@ -609,7 +619,7 @@ namespace ams::ncm {
 
         if (storage_id == StorageId::GameCard) {
             /* Initialize the key value store. */
-            R_TRY(root->kvs->Initialize(root->max_content_metas));
+            R_TRY(root->kvs->Initialize(root->max_content_metas, root->memory_resource->Get()));
 
             /* Create an on memory content meta database for game cards. */
             root->content_meta_database = std::make_shared<OnMemoryContentMetaDatabaseImpl>(std::addressof(*root->kvs));
@@ -621,7 +631,7 @@ namespace ams::ncm {
             auto mount_guard = SCOPE_GUARD { fs::Unmount(root->mount_name); };
 
             /* Initialize and load the key value store from the filesystem. */
-            R_TRY(root->kvs->Initialize(root->path, root->max_content_metas));
+            R_TRY(root->kvs->Initialize(root->path, root->max_content_metas, root->memory_resource->Get()));
             R_TRY(root->kvs->Load());
 
             /* Create the content meta database. */

stratosphere/ncm/source/ncm_main.cpp

@@ -21,7 +21,7 @@ extern "C" {
 
     u32 __nx_applet_type = AppletType_None;
 
-    #define INNER_HEAP_SIZE 0x400000
+    #define INNER_HEAP_SIZE 0x8000
     size_t nx_inner_heap_size = INNER_HEAP_SIZE;
     char   nx_inner_heap[INNER_HEAP_SIZE];
 
@@ -49,6 +49,25 @@ namespace ams {
 
 using namespace ams;
 
+namespace {
+
+    u8 g_heap_memory[1_MB];
+    lmem::HeapHandle g_heap_handle;
+
+    void *Allocate(size_t size) {
+        return lmem::AllocateFromExpHeap(g_heap_handle, size);
+    }
+
+    void Deallocate(void *p, size_t size) {
+        lmem::FreeToExpHeap(g_heap_handle, p);
+    }
+
+    void InitializeHeap() {
+        g_heap_handle = lmem::CreateExpHeap(g_heap_memory, sizeof(g_heap_memory), lmem::CreateOption_None);
+    }
+
+}
+
 void __libnx_exception_handler(ThreadExceptionDump *ctx) {
     ams::CrashHandler(ctx);
 }
@@ -63,11 +82,15 @@ void __libnx_initheap(void) {
 
     fake_heap_start = (char*)addr;
     fake_heap_end   = (char*)addr + size;
+
+    InitializeHeap();
 }
 
 void __appInit(void) {
     hos::SetVersionForLibnx();
 
+    fs::SetAllocator(Allocate, Deallocate);
+
     sm::DoWithSession([&]() {
         R_ABORT_UNLESS(fsInitialize());
         R_ABORT_UNLESS(splInitialize());
@@ -82,6 +105,30 @@ void __appExit(void) {
     fsExit();
 }
 
+void *operator new(size_t size) {
+    return Allocate(size);
+}
+
+void *operator new(size_t size, const std::nothrow_t &) {
+    return Allocate(size);
+}
+
+void operator delete(void *p) {
+    return Deallocate(p, 0);
+}
+
+void *operator new[](size_t size) {
+    return Allocate(size);
+}
+
+void *operator new[](size_t size, const std::nothrow_t &) {
+    return Allocate(size);
+}
+
+void operator delete[](void *p) {
+    return Deallocate(p, 0);
+}
+
 namespace {
 
     struct ContentManagerServerOptions {