mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2024-12-23 02:42:09 +00:00
Code Issues Releases Activity

kern: fuck the KPolice^H^H^H^H^H^HPageGroups

Browse source
This commit is contained in:
Michael Scire 2021-04-07 17:07:01 -07:00 committed by SciresM
parent dc7862882f
commit 96937a611d
8 changed files with 493 additions and 231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp
View file

@ -36,6 +36,10 @@ namespace ams::kern::arch::arm64 {
void Finalize() { m_page_table.Finalize(); }
ALWAYS_INLINE KScopedLightLock AcquireDeviceMapLock() {
return m_page_table.AcquireDeviceMapLock();
}
Result SetMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) {
return m_page_table.SetMemoryPermission(addr, size, perm);
}
@ -148,22 +152,30 @@ namespace ams::kern::arch::arm64 {
return m_page_table.WriteDebugIoMemory(address, buffer, size);
}
Result LockForDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {
return m_page_table.LockForDeviceAddressSpace(out, address, size, perm, is_aligned);
Result LockForMapDeviceAddressSpace(KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {
return m_page_table.LockForMapDeviceAddressSpace(address, size, perm, is_aligned);
}
Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size) {
return m_page_table.LockForUnmapDeviceAddressSpace(address, size);
}
Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size) {
return m_page_table.UnlockForDeviceAddressSpace(address, size);
}
Result MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size) {
return m_page_table.MakePageGroupForUnmapDeviceAddressSpace(out, address, size);
}
Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size) {
return m_page_table.UnlockForDeviceAddressSpacePartialMap(address, size, mapped_size);
}
Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {
return m_page_table.OpenMemoryRangeForMapDeviceAddressSpace(out, address, size, perm, is_aligned);
}
Result OpenMemoryRangeForUnmapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size) {
return m_page_table.OpenMemoryRangeForUnmapDeviceAddressSpace(out, address, size);
}
Result LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size) {
return m_page_table.LockForIpcUserBuffer(out, address, size);
}
@ -188,6 +200,10 @@ namespace ams::kern::arch::arm64 {
return m_page_table.UnlockForCodeMemory(address, size, pg);
}
Result OpenMemoryRangeForProcessCacheOperation(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size) {
return m_page_table.OpenMemoryRangeForProcessCacheOperation(out, address, size);
}
Result CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) {
return m_page_table.CopyMemoryFromLinearToUser(dst_addr, size, src_addr, src_state_mask, src_state, src_test_perm, src_attr_mask, src_attr);
}
@ -240,6 +256,10 @@ namespace ams::kern::arch::arm64 {
return m_page_table.UnmapPhysicalMemoryUnsafe(address, size);
}
Result UnmapProcessMemory(KProcessAddress dst_address, size_t size, KProcessPageTable &src_page_table, KProcessAddress src_address) {
return m_page_table.UnmapProcessMemory(dst_address, size, src_page_table.m_page_table, src_address);
}
void DumpMemoryBlocks() const {
return m_page_table.DumpMemoryBlocks();
}

9
libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp
View file

@ -69,8 +69,8 @@ namespace ams::kern::board::nintendo::nx {
Result Attach(ams::svc::DeviceName device_name, u64 space_address, u64 space_size);
Result Detach(ams::svc::DeviceName device_name);
Result Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings);
Result Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address);
Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings);
Result Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address);
void Unmap(KDeviceVirtualAddress device_address, size_t size) {
return this->UnmapImpl(device_address, size, false);
@ -78,12 +78,11 @@ namespace ams::kern::board::nintendo::nx {
private:
Result MapDevicePage(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm);
Result MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm);
Result MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned);
void UnmapImpl(KDeviceVirtualAddress address, u64 size, bool force);
bool IsFree(KDeviceVirtualAddress address, u64 size) const;
Result MakePageGroup(KPageGroup *out, KDeviceVirtualAddress address, u64 size) const;
bool Compare(const KPageGroup &pg, KDeviceVirtualAddress device_address) const;
bool Compare(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) const;
public:
static void Initialize();

27
libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp
View file

@ -55,6 +55,13 @@ namespace ams::kern {
public:
using TraversalEntry = KPageTableImpl::TraversalEntry;
using TraversalContext = KPageTableImpl::TraversalContext;
struct MemoryRange {
KVirtualAddress address;
size_t size;
void Close();
};
protected:
enum MemoryFillValue {
MemoryFillValue_Zero = 0,
@ -155,6 +162,7 @@ namespace ams::kern {
size_t m_mapped_ipc_server_memory{};
mutable KLightLock m_general_lock{};
mutable KLightLock m_map_physical_memory_lock{};
KLightLock m_device_map_lock{};
KPageTableImpl m_impl{};
KMemoryBlockManager m_memory_block_manager{};
u32 m_allocate_option{};
@ -199,6 +207,10 @@ namespace ams::kern {
return this->CanContain(addr, size, KMemoryState_AliasCode);
}
ALWAYS_INLINE KScopedLightLock AcquireDeviceMapLock() {
return KScopedLightLock(m_device_map_lock);
}
KProcessAddress GetRegionAddress(KMemoryState state) const;
size_t GetRegionSize(KMemoryState state) const;
bool CanContain(KProcessAddress addr, size_t size, KMemoryState state) const;
@ -290,6 +302,8 @@ namespace ams::kern {
Result MakePageGroup(KPageGroup &pg, KProcessAddress addr, size_t num_pages);
bool IsValidPageGroup(const KPageGroup &pg, KProcessAddress addr, size_t num_pages);
Result GetContiguousMemoryRangeWithState(MemoryRange *out, KProcessAddress address, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr);
NOINLINE Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, bool is_pa_valid, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm);
Result MapIoImpl(KProcessAddress *out, PageLinkedList *page_list, KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm);
@ -367,12 +381,15 @@ namespace ams::kern {
Result WriteDebugMemory(KProcessAddress address, const void *buffer, size_t size);
Result WriteDebugIoMemory(KProcessAddress address, const void *buffer, size_t size);
Result LockForDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned);
Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size);
Result LockForMapDeviceAddressSpace(KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned);
Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size);
Result MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size);
Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size);
Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size);
Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned);
Result OpenMemoryRangeForUnmapDeviceAddressSpace(MemoryRange *out, KProcessAddress address, size_t size);
Result LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size);
Result UnlockForIpcUserBuffer(KProcessAddress address, size_t size);
@ -381,6 +398,8 @@ namespace ams::kern {
Result LockForCodeMemory(KPageGroup *out, KProcessAddress address, size_t size);
Result UnlockForCodeMemory(KProcessAddress address, size_t size, const KPageGroup &pg);
Result OpenMemoryRangeForProcessCacheOperation(MemoryRange *out, KProcessAddress address, size_t size);
Result CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr);
Result CopyMemoryFromLinearToKernel(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr);
Result CopyMemoryFromUserToLinear(KProcessAddress dst_addr, size_t size, u32 dst_state_mask, u32 dst_state, KMemoryPermission dst_test_perm, u32 dst_attr_mask, u32 dst_attr, KProcessAddress src_addr);
@ -398,6 +417,8 @@ namespace ams::kern {
Result MapPhysicalMemoryUnsafe(KProcessAddress address, size_t size);
Result UnmapPhysicalMemoryUnsafe(KProcessAddress address, size_t size);
Result UnmapProcessMemory(KProcessAddress dst_address, size_t size, KPageTableBase &src_pt, KProcessAddress src_address);
void DumpMemoryBlocksLocked() const {
MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
m_memory_block_manager.DumpBlocks();

280
libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp
View file

@ -216,6 +216,12 @@ namespace ams::kern::board::nintendo::nx {
return (m_value & (1u << n));
}
template<Bit... Bits>
constexpr ALWAYS_INLINE u32 SelectBits() const {
constexpr u32 Mask = ((1u << Bits) | ...);
return m_value & Mask;
}
constexpr ALWAYS_INLINE bool GetBit(Bit n) const {
return this->SelectBit(n) != 0;
}
@ -242,12 +248,14 @@ namespace ams::kern::board::nintendo::nx {
constexpr ALWAYS_INLINE bool IsNonSecure() const { return this->GetBit(Bit_NonSecure); }
constexpr ALWAYS_INLINE bool IsWriteable() const { return this->GetBit(Bit_Writeable); }
constexpr ALWAYS_INLINE bool IsReadable() const { return this->GetBit(Bit_Readable); }
constexpr ALWAYS_INLINE bool IsValid() const { return this->IsWriteable() || this->IsReadable(); }
constexpr ALWAYS_INLINE bool IsValid() const { return this->SelectBits<Bit_Readable, Bit_Writeable>(); }
constexpr ALWAYS_INLINE u32 GetAttributes() const { return this->SelectBit(Bit_NonSecure) | this->SelectBit(Bit_Writeable) | this->SelectBit(Bit_Readable); }
constexpr ALWAYS_INLINE u32 GetAttributes() const { return this->SelectBits<Bit_Readable, Bit_Writeable, Bit_NonSecure>(); }
constexpr ALWAYS_INLINE KPhysicalAddress GetPhysicalAddress() const { return (static_cast<u64>(m_value) << DevicePageBits) & PhysicalAddressMask; }
ALWAYS_INLINE void InvalidateAttributes() { this->SetValue(m_value & ~(0xCu << 28)); }
ALWAYS_INLINE void Invalidate() { this->SetValue(0); }
};
@ -847,7 +855,7 @@ namespace ams::kern::board::nintendo::nx {
}
/* Forcibly unmap all pages. */
this->UnmapImpl(0, (1ul << DeviceVirtualAddressBits), true);
this->UnmapImpl(0, (1ul << DeviceVirtualAddressBits), false);
/* Release all asids. */
for (size_t i = 0; i < TableCount; ++i) {
@ -1117,12 +1125,11 @@ namespace ams::kern::board::nintendo::nx {
return ResultSuccess();
}
Result KDevicePageTable::MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm) {
Result KDevicePageTable::MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned) {
/* Clear the output size. */
*out_mapped_size = 0;
/* Get the size, and validate the address. */
const u64 size = pg.GetNumPages() * PageSize;
MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
@ -1130,28 +1137,33 @@ namespace ams::kern::board::nintendo::nx {
R_UNLESS(this->IsFree(device_address, size), svc::ResultInvalidCurrentMemory());
/* Ensure that if we fail, we unmap anything we mapped. */
auto unmap_guard = SCOPE_GUARD { this->UnmapImpl(device_address, size, true); };
auto unmap_guard = SCOPE_GUARD { this->UnmapImpl(device_address, size, false); };
/* Iterate, mapping device pages. */
KDeviceVirtualAddress cur_addr = device_address;
for (auto it = pg.begin(); it != pg.end(); ++it) {
/* Require that we be able to map the device page. */
R_UNLESS(IsHeapVirtualAddress(it->GetAddress()), svc::ResultInvalidCurrentMemory());
while (true) {
/* Get the current contiguous range. */
KPageTableBase::MemoryRange contig_range = {};
R_TRY(page_table->OpenMemoryRangeForMapDeviceAddressSpace(std::addressof(contig_range), process_address + *out_mapped_size, size - *out_mapped_size, ConvertToKMemoryPermission(device_perm), is_aligned));
/* Get the physical address for the page. */
const KPhysicalAddress phys_addr = GetHeapPhysicalAddress(it->GetAddress());
/* Ensure we close the range when we're done. */
ON_SCOPE_EXIT { contig_range.Close(); };
/* Map the device page. */
const u64 block_size = it->GetSize();
size_t mapped_size = 0;
R_TRY(this->MapDevicePage(std::addressof(mapped_size), num_pt, max_pt, phys_addr, block_size, cur_addr, device_perm));
R_TRY(this->MapDevicePage(std::addressof(mapped_size), num_pt, max_pt, GetHeapPhysicalAddress(contig_range.address), contig_range.size, cur_addr, device_perm));
/* Advance. */
cur_addr += block_size;
cur_addr += contig_range.size;
*out_mapped_size += mapped_size;
/* If we didn't map as much as we wanted, break. */
if (mapped_size < block_size) {
if (mapped_size < contig_range.size) {
break;
}
/* Similarly, if we're done, break. */
if (*out_mapped_size >= size) {
break;
}
}
@ -1186,8 +1198,6 @@ namespace ams::kern::board::nintendo::nx {
/* Check if there's nothing mapped at l1. */
if (l1 == nullptr || !l1[l1_index].IsValid()) {
MESOSPHERE_ASSERT(force);
const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index;
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining / DevicePageSize);
@ -1201,30 +1211,12 @@ namespace ams::kern::board::nintendo::nx {
const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index;
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining / DevicePageSize);
size_t num_closed = 0;
bool invalidated_tlb = false;
/* Invalidate the attributes of all entries. */
for (size_t i = 0; i < map_count; ++i) {
if (l2[l2_index + i].IsValid()) {
/* Get the physical address. */
const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Invalidate the entry. */
l2[l2_index + i].Invalidate();
l2[l2_index + i].InvalidateAttributes();
++num_closed;
/* Try to add the page to the group. */
if (R_FAILED(pg.AddBlock(GetHeapVirtualAddress(phys_addr), DevicePageSize / PageSize))) {
/* If we can't add it for deferred close, close it now. */
cpu::StoreDataCache(std::addressof(l2[l2_index + i]), sizeof(PageTableEntry));
InvalidatePtc(GetPageTablePhysicalAddress(KVirtualAddress(std::addressof(l2[l2_index + i]))));
SmmuSynchronizationBarrier();
/* Close the page's reference. */
mm.Close(GetHeapVirtualAddress(phys_addr), 1);
}
} else {
MESOSPHERE_ASSERT(force);
}
}
cpu::StoreDataCache(std::addressof(l2[l2_index]), map_count * sizeof(PageTableEntry));
@ -1235,6 +1227,38 @@ namespace ams::kern::board::nintendo::nx {
}
SmmuSynchronizationBarrier();
/* Close the memory manager's references to the pages. */
{
KPhysicalAddress contig_phys_addr = Null<KPhysicalAddress>;
size_t contig_count = 0;
for (size_t i = 0; i < map_count; ++i) {
/* Get the physical address. */
const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Fully invalidate the entry. */
l2[l2_index + i].Invalidate();
if (contig_count == 0) {
/* Ensure that our address/count is valid. */
contig_phys_addr = phys_addr;
contig_count = contig_phys_addr != Null<KPhysicalAddress> ? 1 : 0;
} else if (phys_addr == Null<KPhysicalAddress> || phys_addr != (contig_phys_addr + (contig_count * DevicePageSize))) {
/* If we're no longer contiguous, close the range we've been building. */
mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize);
contig_phys_addr = phys_addr;
contig_count = contig_phys_addr != Null<KPhysicalAddress> ? 1 : 0;
} else {
++contig_count;
}
}
if (contig_count > 0) {
mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize);
}
}
/* Close the pages. */
if (ptm.Close(KVirtualAddress(l2), num_closed)) {
/* Invalidate the l1 entry. */
@ -1243,22 +1267,12 @@ namespace ams::kern::board::nintendo::nx {
/* Synchronize. */
InvalidatePtc(GetPageTablePhysicalAddress(KVirtualAddress(std::addressof(l1[l1_index]))));
InvalidateTlbSection(m_table_asids[l0_index], address);
SmmuSynchronizationBarrier();
/* We invalidated the tlb. */
invalidated_tlb = true;
/* Free the l2 page. */
ptm.Free(KVirtualAddress(l2));
}
/* Invalidate the tlb if we haven't already. */
if (!invalidated_tlb) {
InvalidateTlbSection(m_table_asids[l0_index], address);
SmmuSynchronizationBarrier();
}
/* Advance. */
address += map_count * DevicePageSize;
remaining -= map_count * DevicePageSize;
@ -1287,114 +1301,158 @@ namespace ams::kern::board::nintendo::nx {
remaining -= DeviceLargePageSize;
}
}
/* Close references to the pages in the group. */
pg.Close();
}
Result KDevicePageTable::MakePageGroup(KPageGroup *out, KDeviceVirtualAddress address, u64 size) const {
MESOSPHERE_ASSERT((address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((address + size - 1) & ~DeviceVirtualAddressMask) == 0);
bool KDevicePageTable::Compare(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) const {
MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
/* We need to traverse the ranges that make up our mapping, to make sure they're all good. Start by getting a contiguous range. */
KPageTableBase::MemoryRange contig_range = {};
if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), process_address, size))) {
return false;
}
/* Ensure that we close the range when we're done. */
bool range_open = true;
ON_SCOPE_EXIT { if (range_open) { contig_range.Close(); } };
/* Walk the directory. */
u64 remaining = size;
bool first = true;
u32 attr = 0;
while (remaining > 0) {
const size_t l0_index = (address / DeviceRegionSize);
const size_t l1_index = (address % DeviceRegionSize) / DeviceLargePageSize;
const size_t l2_index = (address % DeviceLargePageSize) / DevicePageSize;
KProcessAddress cur_process_address = process_address;
size_t remaining_size = size;
KPhysicalAddress cur_phys_address = GetHeapPhysicalAddress(contig_range.address);
size_t remaining_in_range = contig_range.size;
bool first = true;
u32 first_attr = 0;
while (remaining_size > 0) {
/* Convert the device address to a series of indices. */
const size_t l0_index = (device_address / DeviceRegionSize);
const size_t l1_index = (device_address % DeviceRegionSize) / DeviceLargePageSize;
const size_t l2_index = (device_address % DeviceLargePageSize) / DevicePageSize;
/* Get and validate l1. */
const PageDirectoryEntry *l1 = GetPointer<PageDirectoryEntry>(m_tables[l0_index]);
R_UNLESS(l1 != nullptr, svc::ResultInvalidCurrentMemory());
R_UNLESS(l1[l1_index].IsValid(), svc::ResultInvalidCurrentMemory());
if (!(l1 != nullptr && l1[l1_index].IsValid())) {
return false;
}
if (l1[l1_index].IsTable()) {
/* We're acting on an l2 entry. */
const PageTableEntry *l2 = GetPointer<PageTableEntry>(GetPageTableVirtualAddress(l1[l1_index].GetPhysicalAddress()));
/* Determine the number of pages to check. */
const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index;
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining / DevicePageSize);
const size_t map_count = std::min<size_t>(remaining_in_entry, remaining_size / DevicePageSize);
/* Check each page. */
for (size_t i = 0; i < map_count; ++i) {
/* Ensure the l2 entry is valid. */
R_UNLESS(l2[l2_index + i].IsValid(), svc::ResultInvalidCurrentMemory());
/* Get the physical address. */
const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Add to the group. */
R_TRY(out->AddBlock(GetHeapVirtualAddress(phys_addr), DevicePageSize / PageSize));
/* If this is our first entry, get the attribute. */
if (first) {
attr = l2[l2_index + i].GetAttributes();
first = false;
} else {
/* Validate the attributes match the first entry. */
R_UNLESS(l2[l2_index + i].GetAttributes() == attr, svc::ResultInvalidCurrentMemory());
if (!l2[l2_index + i].IsValid()) {
return false;
}
/* Check that the attributes match the first attributes we encountered. */
const u32 cur_attr = l2[l2_index + i].GetAttributes();
if (!first && cur_attr != first_attr) {
return false;
}
/* If there's nothing remaining in the range, refresh the range. */
if (remaining_in_range == 0) {
contig_range.Close();
range_open = false;
if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), cur_process_address, remaining_size))) {
return false;
}
range_open = true;
cur_phys_address = GetHeapPhysicalAddress(contig_range.address);
remaining_in_range = contig_range.size;
}
/* Check that the physical address is expected. */
if (l2[l2_index + i].GetPhysicalAddress() != cur_phys_address) {
return false;
}
/* Advance. */
cur_phys_address += DevicePageSize;
cur_process_address += DevicePageSize;
remaining_size -= DevicePageSize;
remaining_in_range -= DevicePageSize;
first = false;
first_attr = cur_attr;
}
/* Advance. */
address += DevicePageSize * map_count;
remaining -= DevicePageSize * map_count;
/* Advance the device address. */
device_address += map_count * DevicePageSize;
} else {
/* We're acting on an l1 entry. */
R_UNLESS(l2_index == 0, svc::ResultInvalidCurrentMemory());
R_UNLESS(remaining >= DeviceLargePageSize, svc::ResultInvalidCurrentMemory());
if (!(l2_index == 0 && remaining_size >= DeviceLargePageSize)) {
return false;
}
/* Get the physical address. */
const KPhysicalAddress phys_addr = l1[l1_index].GetPhysicalAddress();
MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr));
/* Check that the attributes match the first attributes we encountered. */
const u32 cur_attr = l1[l1_index].GetAttributes();
if (!first && cur_attr != first_attr) {
return false;
}
/* Add to the group. */
R_TRY(out->AddBlock(GetHeapVirtualAddress(phys_addr), DeviceLargePageSize / PageSize));
/* If there's nothing remaining in the range, refresh the range. */
if (remaining_in_range == 0) {
contig_range.Close();
/* If this is our first entry, get the attribute. */
if (first) {
attr = l1[l1_index].GetAttributes();
first = false;
} else {
/* Validate the attributes match the first entry. */
R_UNLESS(l1[l1_index].GetAttributes() == attr, svc::ResultInvalidCurrentMemory());
range_open = false;
if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), cur_process_address, remaining_size))) {
return false;
}
range_open = true;
cur_phys_address = GetHeapPhysicalAddress(contig_range.address);
remaining_in_range = contig_range.size;
}
/* Check that the physical address is expected, and there's enough in the range. */
if (remaining_in_range < DeviceLargePageSize || l1[l1_index].GetPhysicalAddress() != cur_phys_address) {
return false;
}
/* Advance. */
address += DeviceLargePageSize;
remaining -= DeviceLargePageSize;
cur_phys_address += DeviceLargePageSize;
cur_process_address += DeviceLargePageSize;
remaining_size -= DeviceLargePageSize;
remaining_in_range -= DeviceLargePageSize;
first = false;
first_attr = cur_attr;
/* Advance the device address. */
device_address += DeviceLargePageSize;
}
}
return ResultSuccess();
/* The range is valid! */
return true;
}
bool KDevicePageTable::Compare(const KPageGroup &compare_pg, KDeviceVirtualAddress device_address) const {
/* Check whether the page group we expect for the virtual address matches the page group we're validating. */
KPageGroup calc_pg(std::addressof(Kernel::GetBlockInfoManager()));
return (R_SUCCEEDED(this->MakePageGroup(std::addressof(calc_pg), device_address, compare_pg.GetNumPages() * PageSize))) &&
calc_pg.IsEquivalentTo(compare_pg);
}
Result KDevicePageTable::Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) {
Result KDevicePageTable::Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) {
/* Clear the output size. */
*out_mapped_size = 0;
/* Map the pages. */
s32 num_pt = 0;
return this->MapImpl(out_mapped_size, num_pt, refresh_mappings ? 1 : std::numeric_limits<s32>::max(), pg, device_address, device_perm);
return this->MapImpl(out_mapped_size, num_pt, refresh_mappings ? 1 : std::numeric_limits<s32>::max(), page_table, process_address, size, device_address, device_perm, refresh_mappings);
}
Result KDevicePageTable::Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address) {
Result KDevicePageTable::Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) {
/* Validate address/size. */
const size_t size = pg.GetNumPages() * PageSize;
MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0);
MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0);
/* Ensure the page group is correct. */
R_UNLESS(this->Compare(pg, device_address), svc::ResultInvalidCurrentMemory());
R_UNLESS(this->Compare(page_table, process_address, size, device_address), svc::ResultInvalidCurrentMemory());
/* Unmap the pages. */
this->UnmapImpl(device_address, size, false);

22
libraries/libmesosphere/source/kern_k_device_address_space.cpp
View file

@ -71,12 +71,11 @@ namespace ams::kern {
/* Lock the address space. */
KScopedLightLock lk(m_lock);
/* Lock the pages. */
KPageGroup pg(page_table->GetBlockInfoManager());
R_TRY(page_table->LockForDeviceAddressSpace(std::addressof(pg), process_address, size, ConvertToKMemoryPermission(device_perm), is_aligned));
/* Lock the page table to prevent concurrent device mapping operations. */
KScopedLightLock pt_lk = page_table->AcquireDeviceMapLock();
/* Close the pages we opened when we're done with them. */
ON_SCOPE_EXIT { pg.Close(); };
/* Lock the pages. */
R_TRY(page_table->LockForMapDeviceAddressSpace(process_address, size, ConvertToKMemoryPermission(device_perm), is_aligned));
/* Ensure that if we fail, we don't keep unmapped pages locked. */
auto unlock_guard = SCOPE_GUARD { MESOSPHERE_R_ABORT_UNLESS(page_table->UnlockForDeviceAddressSpace(process_address, size)); };
@ -87,7 +86,7 @@ namespace ams::kern {
auto mapped_size_guard = SCOPE_GUARD { *out_mapped_size = 0; };
/* Perform the mapping. */
R_TRY(m_table.Map(out_mapped_size, pg, device_address, device_perm, refresh_mappings));
R_TRY(m_table.Map(out_mapped_size, page_table, process_address, size, device_address, device_perm, refresh_mappings));
/* Ensure that we unmap the pages if we fail to update the protections. */
/* NOTE: Nintendo does not check the result of this unmap call. */
@ -113,19 +112,18 @@ namespace ams::kern {
/* Lock the address space. */
KScopedLightLock lk(m_lock);
/* Make and open a page group for the unmapped region. */
KPageGroup pg(page_table->GetBlockInfoManager());
R_TRY(page_table->MakePageGroupForUnmapDeviceAddressSpace(std::addressof(pg), process_address, size));
/* Lock the page table to prevent concurrent device mapping operations. */
KScopedLightLock pt_lk = page_table->AcquireDeviceMapLock();
/* Ensure the page group is closed on scope exit. */
ON_SCOPE_EXIT { pg.Close(); };
/* Lock the pages. */
R_TRY(page_table->LockForUnmapDeviceAddressSpace(process_address, size));
/* If we fail to unmap, we want to do a partial unlock. */
{
auto unlock_guard = SCOPE_GUARD { page_table->UnlockForDeviceAddressSpacePartialMap(process_address, size, size); };
/* Unmap. */
R_TRY(m_table.Unmap(pg, device_address));
R_TRY(m_table.Unmap(page_table, process_address, size, device_address));
unlock_guard.Cancel();
}

305
libraries/libmesosphere/source/kern_k_page_table_base.cpp
View file

@ -74,6 +74,10 @@ namespace ams::kern {
}
void KPageTableBase::MemoryRange::Close() {
Kernel::GetMemoryManager().Close(address, size / PageSize);
}
Result KPageTableBase::InitializeForKernel(bool is_64_bit, void *table, KVirtualAddress start, KVirtualAddress end) {
/* Initialize our members. */
m_address_space_width = (is_64_bit) ? BITSIZEOF(u64) : BITSIZEOF(u32);
@ -1391,6 +1395,49 @@ namespace ams::kern {
return cur_block_address == GetHeapVirtualAddress(cur_addr) && cur_block_pages == (cur_size / PageSize);
}
Result KPageTableBase::GetContiguousMemoryRangeWithState(MemoryRange *out, KProcessAddress address, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) {
MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
auto &impl = this->GetImpl();
/* Begin a traversal. */
TraversalContext context;
TraversalEntry cur_entry = {};
R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address), svc::ResultInvalidCurrentMemory());
/* The region we're traversing has to be heap. */
const KPhysicalAddress phys_address = cur_entry.phys_addr;
R_UNLESS(this->IsHeapPhysicalAddress(phys_address), svc::ResultInvalidCurrentMemory());
/* Traverse until we have enough size or we aren't contiguous any more. */
size_t contig_size;
for (contig_size = cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1)); contig_size < size; contig_size += cur_entry.block_size) {
if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) {
break;
}
if (cur_entry.phys_addr != phys_address + contig_size) {
break;
}
}
/* Take the minimum size for our region. */
size = std::min(size, contig_size);
/* Check that the memory is contiguous. */
R_TRY(this->CheckMemoryStateContiguous(address, size,
state_mask | KMemoryState_FlagReferenceCounted, state | KMemoryState_FlagReferenceCounted,
perm_mask, perm,
attr_mask, attr));
/* The memory is contiguous, so set the output range. */
*out = {
.address = GetLinearMappedVirtualAddress(phys_address),
.size = size,
};
return ResultSuccess();
}
Result KPageTableBase::SetMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission svc_perm) {
const size_t num_pages = size / PageSize;
@ -2578,7 +2625,7 @@ namespace ams::kern {
return ResultSuccess();
}
Result KPageTableBase::LockForDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {
Result KPageTableBase::LockForMapDeviceAddressSpace(KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {
/* Lightly validate the range before doing anything else. */
const size_t num_pages = size / PageSize;
R_UNLESS(this->Contains(address, size), svc::ResultInvalidCurrentMemory());
@ -2591,11 +2638,6 @@ namespace ams::kern {
size_t num_allocator_blocks;
R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, test_state, test_state, perm, perm, KMemoryAttribute_IpcLocked | KMemoryAttribute_Locked, KMemoryAttribute_None, KMemoryAttribute_DeviceShared));
/* Make the page group, if we should. */
if (out != nullptr) {
R_TRY(this->MakePageGroup(*out, address, num_pages));
}
/* Create an update allocator. */
Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks);
@ -2604,10 +2646,33 @@ namespace ams::kern {
/* Update the memory blocks. */
m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, &KMemoryBlock::ShareToDevice, KMemoryPermission_None);
/* Open the page group. */
if (out != nullptr) {
out->Open();
}
return ResultSuccess();
}
Result KPageTableBase::LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size) {
/* Lightly validate the range before doing anything else. */
const size_t num_pages = size / PageSize;
R_UNLESS(this->Contains(address, size), svc::ResultInvalidCurrentMemory());
/* Lock the table. */
KScopedLightLock lk(m_general_lock);
/* Check the memory state. */
size_t num_allocator_blocks;
R_TRY(this->CheckMemoryStateContiguous(std::addressof(num_allocator_blocks),
address, size,
KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap, KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
/* Create an update allocator. */
Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks);
R_TRY(allocator_result);
/* Update the memory blocks. */
const KMemoryBlockManager::MemoryBlockLockFunction lock_func = m_enable_device_address_space_merge ? &KMemoryBlock::UpdateDeviceDisableMergeStateForShare : &KMemoryBlock::UpdateDeviceDisableMergeStateForShareRight;
m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, lock_func, KMemoryPermission_None);
return ResultSuccess();
}
@ -2639,40 +2704,6 @@ namespace ams::kern {
return ResultSuccess();
}
Result KPageTableBase::MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size) {
/* Lightly validate the range before doing anything else. */
const size_t num_pages = size / PageSize;
R_UNLESS(this->Contains(address, size), svc::ResultInvalidCurrentMemory());
/* Lock the table. */
KScopedLightLock lk(m_general_lock);
/* Check the memory state. */
size_t num_allocator_blocks;
R_TRY(this->CheckMemoryStateContiguous(std::addressof(num_allocator_blocks),
address, size,
KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap, KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
/* Create an update allocator. */
Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks);
R_TRY(allocator_result);
/* Make the page group. */
R_TRY(this->MakePageGroup(*out, address, num_pages));
/* Update the memory blocks. */
const KMemoryBlockManager::MemoryBlockLockFunction lock_func = m_enable_device_address_space_merge ? &KMemoryBlock::UpdateDeviceDisableMergeStateForShare : &KMemoryBlock::UpdateDeviceDisableMergeStateForShareRight;
m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, lock_func, KMemoryPermission_None);
/* Open a reference to the pages in the page group. */
out->Open();
return ResultSuccess();
}
Result KPageTableBase::UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size) {
/* Lightly validate the range before doing anything else. */
const size_t num_pages = size / PageSize;
@ -2689,23 +2720,23 @@ namespace ams::kern {
size_t allocator_num_blocks = 0, unmapped_allocator_num_blocks = 0;
if (unmapped_size) {
if (m_enable_device_address_space_merge) {
R_TRY(this->CheckMemoryState(std::addressof(allocator_num_blocks),
address, size,
KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
R_TRY(this->CheckMemoryStateContiguous(std::addressof(allocator_num_blocks),
address, size,
KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
}
R_TRY(this->CheckMemoryState(std::addressof(unmapped_allocator_num_blocks),
mapped_end_address, unmapped_size,
KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
R_TRY(this->CheckMemoryStateContiguous(std::addressof(unmapped_allocator_num_blocks),
mapped_end_address, unmapped_size,
KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
} else {
R_TRY(this->CheckMemoryState(std::addressof(allocator_num_blocks),
address, size,
KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
R_TRY(this->CheckMemoryStateContiguous(std::addressof(allocator_num_blocks),
address, size,
KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
}
/* Create an update allocator for the region. */
@ -2750,6 +2781,41 @@ namespace ams::kern {
return ResultSuccess();
}
Result KPageTableBase::OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) {
/* Lock the table. */
KScopedLightLock lk(m_general_lock);
/* Get the range. */
const u32 test_state = KMemoryState_FlagReferenceCounted | (is_aligned ? KMemoryState_FlagCanAlignedDeviceMap : KMemoryState_FlagCanDeviceMap);
R_TRY(this->GetContiguousMemoryRangeWithState(out,
address, size,
test_state, test_state,
perm, perm,
KMemoryAttribute_IpcLocked | KMemoryAttribute_Locked, KMemoryAttribute_None));
/* We got the range, so open it. */
Kernel::GetMemoryManager().Open(out->address, out->size / PageSize);
return ResultSuccess();
}
Result KPageTableBase::OpenMemoryRangeForUnmapDeviceAddressSpace(MemoryRange *out, KProcessAddress address, size_t size) {
/* Lock the table. */
KScopedLightLock lk(m_general_lock);
/* Get the range. */
R_TRY(this->GetContiguousMemoryRangeWithState(out,
address, size,
KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared));
/* We got the range, so open it. */
Kernel::GetMemoryManager().Open(out->address, out->size / PageSize);
return ResultSuccess();
}
Result KPageTableBase::LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size) {
return this->LockMemoryAndOpen(nullptr, out, address, size,
KMemoryState_FlagCanIpcUserBuffer, KMemoryState_FlagCanIpcUserBuffer,
@ -2804,6 +2870,23 @@ namespace ams::kern {
KMemoryAttribute_Locked, std::addressof(pg));
}
Result KPageTableBase::OpenMemoryRangeForProcessCacheOperation(MemoryRange *out, KProcessAddress address, size_t size) {
/* Lock the table. */
KScopedLightLock lk(m_general_lock);
/* Get the range. */
R_TRY(this->GetContiguousMemoryRangeWithState(out,
address, size,
KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted,
KMemoryPermission_UserRead, KMemoryPermission_UserRead,
KMemoryAttribute_Uncached, KMemoryAttribute_None));
/* We got the range, so open it. */
Kernel::GetMemoryManager().Open(out->address, out->size / PageSize);
return ResultSuccess();
}
Result KPageTableBase::CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) {
/* Lightly validate the range before doing anything else. */
R_UNLESS(this->Contains(src_addr, size), svc::ResultInvalidCurrentMemory());
@ -4553,4 +4636,108 @@ namespace ams::kern {
return ResultSuccess();
}
Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t size, KPageTableBase &src_page_table, KProcessAddress src_address) {
/* We need to lock both this table, and the current process's table, so set up an alias. */
KPageTableBase &dst_page_table = *this;
/* Acquire the table locks. */
KScopedLightLockPair lk(src_page_table.m_general_lock, dst_page_table.m_general_lock);
/* Check that the memory is mapped in the destination process. */
size_t num_allocator_blocks;
R_TRY(dst_page_table.CheckMemoryState(std::addressof(num_allocator_blocks), dst_address, size, KMemoryState_All, KMemoryState_SharedCode, KMemoryPermission_UserReadWrite, KMemoryPermission_UserReadWrite, KMemoryAttribute_All, KMemoryAttribute_None));
/* Check that the memory is mapped in the source process. */
R_TRY(src_page_table.CheckMemoryState(src_address, size, KMemoryState_FlagCanMapProcess, KMemoryState_FlagCanMapProcess, KMemoryPermission_None, KMemoryPermission_None, KMemoryAttribute_All, KMemoryAttribute_None));
/* Validate that the memory ranges are compatible. */
{
/* Define a helper type. */
struct ContiguousRangeInfo {
public:
KPageTableBase &m_pt;
TraversalContext m_context;
TraversalEntry m_entry;
KPhysicalAddress m_phys_addr;
size_t m_cur_size;
size_t m_remaining_size;
public:
ContiguousRangeInfo(KPageTableBase &pt, KProcessAddress address, size_t size) : m_pt(pt), m_remaining_size(size) {
/* Begin a traversal. */
MESOSPHERE_ABORT_UNLESS(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry), std::addressof(m_context), address));
/* Setup tracking fields. */
m_phys_addr = m_entry.phys_addr;
m_cur_size = std::min<size_t>(m_remaining_size, m_entry.block_size - (GetInteger(m_phys_addr) & (m_entry.block_size - 1)));
/* Consume the whole contiguous block. */
this->DetermineContiguousBlockExtents();
}
void ContinueTraversal() {
/* Update our remaining size. */
m_remaining_size = m_remaining_size - m_cur_size;
/* Update our tracking fields. */
if (m_remaining_size > 0) {
m_phys_addr = m_entry.phys_addr;
m_cur_size = std::min<size_t>(m_remaining_size, m_entry.block_size);
/* Consume the whole contiguous block. */
this->DetermineContiguousBlockExtents();
}
}
private:
void DetermineContiguousBlockExtents() {
/* Continue traversing until we're not contiguous, or we have enough. */
while (m_cur_size < m_remaining_size) {
MESOSPHERE_ABORT_UNLESS(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry), std::addressof(m_context)));
/* If we're not contiguous, we're done. */
if (m_entry.phys_addr != m_phys_addr + m_cur_size) {
break;
}
/* Update our current size. */
m_cur_size = std::min(m_remaining_size, m_cur_size + m_entry.block_size);
}
}
};
/* Create ranges for both tables. */
ContiguousRangeInfo src_range(src_page_table, src_address, size);
ContiguousRangeInfo dst_range(dst_page_table, dst_address, size);
/* Validate the ranges. */
while (src_range.m_remaining_size > 0 && dst_range.m_remaining_size > 0) {
R_UNLESS(src_range.m_phys_addr == dst_range.m_phys_addr, svc::ResultInvalidMemoryRegion());
R_UNLESS(src_range.m_cur_size == dst_range.m_cur_size, svc::ResultInvalidMemoryRegion());
src_range.ContinueTraversal();
dst_range.ContinueTraversal();
}
}
/* We no longer need to hold our lock on the source page table. */
lk.TryUnlockHalf(src_page_table.m_general_lock);
/* Create an update allocator. */
Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks);
R_TRY(allocator_result);
/* We're going to perform an update, so create a helper. */
KScopedPageTableUpdater updater(this);
/* Unmap the memory. */
const size_t num_pages = size / PageSize;
const KPageProperties unmap_properties = { KMemoryPermission_None, false, false, DisableMergeAttribute_None };
R_TRY(this->Operate(updater.GetPageList(), dst_address, num_pages, Null<KPhysicalAddress>, false, unmap_properties, OperationType_Unmap, false));
/* Apply the memory block update. */
m_memory_block_manager.Update(std::addressof(allocator), dst_address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal);
return ResultSuccess();
}
}

32
libraries/libmesosphere/source/svc/kern_svc_cache.cpp
View file

@ -30,32 +30,24 @@ namespace ams::kern::svc {
/* Determine aligned extents. */
const uintptr_t aligned_start = util::AlignDown(address, PageSize);
const uintptr_t aligned_end = util::AlignUp(address + size, PageSize);
const size_t num_pages = (aligned_end - aligned_start) / PageSize;
/* Create a page group for the process's memory. */
KPageGroup pg(page_table.GetBlockInfoManager());
/* Make and open the page group. */
R_TRY(page_table.MakeAndOpenPageGroup(std::addressof(pg),
aligned_start, num_pages,
KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted,
KMemoryPermission_UserRead, KMemoryPermission_UserRead,
KMemoryAttribute_Uncached, KMemoryAttribute_None));
/* Ensure we don't leak references to the pages we're operating on. */
ON_SCOPE_EXIT { pg.Close(); };
/* Operate on all the blocks. */
/* Iterate over and operate on contiguous ranges. */
uintptr_t cur_address = aligned_start;
size_t remaining = size;
for (const auto &block : pg) {
/* Get the block extents. */
KVirtualAddress operate_address = block.GetAddress();
size_t operate_size = block.GetSize();
while (remaining > 0) {
/* Get a contiguous range to operate on. */
KPageTableBase::MemoryRange contig_range = {};
R_TRY(page_table.OpenMemoryRangeForProcessCacheOperation(std::addressof(contig_range), cur_address, aligned_end - cur_address));
/* Close the range when we're done operating on it. */
ON_SCOPE_EXIT { contig_range.Close(); };
/* Adjust to remain within range. */
KVirtualAddress operate_address = contig_range.address;
size_t operate_size = contig_range.size;
if (cur_address < address) {
operate_address += (address - cur_address);
operate_size -= (address - cur_address);
}
if (operate_size > remaining) {
operate_size = remaining;
@ -65,7 +57,7 @@ namespace ams::kern::svc {
operation.Operate(GetVoidPointer(operate_address), operate_size);
/* Advance. */
cur_address += block.GetSize();
cur_address += contig_range.size;
remaining -= operate_size;
}
MESOSPHERE_ASSERT(remaining == 0);

17
libraries/libmesosphere/source/svc/kern_svc_process_memory.cpp
View file

@ -122,21 +122,8 @@ namespace ams::kern::svc {
R_UNLESS(src_pt.Contains(src_address, size), svc::ResultInvalidCurrentMemory());
R_UNLESS(dst_pt.CanContain(dst_address, size, KMemoryState_SharedCode), svc::ResultInvalidMemoryRegion());
/* Create a new page group. */
KPageGroup pg(dst_pt.GetBlockInfoManager());
/* Make the page group. */
R_TRY(src_pt.MakeAndOpenPageGroup(std::addressof(pg),
src_address, size / PageSize,
KMemoryState_FlagCanMapProcess, KMemoryState_FlagCanMapProcess,
KMemoryPermission_None, KMemoryPermission_None,
KMemoryAttribute_All, KMemoryAttribute_None));
/* Close the page group when we're done. */
ON_SCOPE_EXIT { pg.Close(); };
/* Unmap the group. */
R_TRY(dst_pt.UnmapPageGroup(dst_address, pg, KMemoryState_SharedCode));
/* Unmap the memory. */
R_TRY(dst_pt.UnmapProcessMemory(dst_address, size, src_pt, src_address));
return ResultSuccess();
}