using Ryujinx.Common.Memory.PartialUnmaps;
using System;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.Versioning;
using System.Threading;
namespace Ryujinx.Memory.WindowsShared
{
///
/// Windows memory placeholder manager.
///
[SupportedOSPlatform("windows")]
class PlaceholderManager
{
private const int InitialOverlapsSize = 10;
private readonly MappingTree _mappings;
private readonly MappingTree _protections;
private readonly IntPtr _partialUnmapStatePtr;
private readonly Thread _partialUnmapTrimThread;
///
/// Creates a new instance of the Windows memory placeholder manager.
///
public PlaceholderManager()
{
_mappings = new MappingTree();
_protections = new MappingTree();
_partialUnmapStatePtr = PartialUnmapState.GlobalState;
_partialUnmapTrimThread = new Thread(TrimThreadLocalMapLoop);
_partialUnmapTrimThread.Name = "CPU.PartialUnmapTrimThread";
_partialUnmapTrimThread.IsBackground = true;
_partialUnmapTrimThread.Start();
}
///
/// Gets a reference to the partial unmap state struct.
///
/// A reference to the partial unmap state struct
private unsafe ref PartialUnmapState GetPartialUnmapState()
{
return ref Unsafe.AsRef((void*)_partialUnmapStatePtr);
}
///
/// Trims inactive threads from the partial unmap state's thread mapping every few seconds.
/// Should be run in a Background thread so that it doesn't stop the program from closing.
///
private void TrimThreadLocalMapLoop()
{
while (true)
{
Thread.Sleep(2000);
GetPartialUnmapState().TrimThreads();
}
}
///
/// Reserves a range of the address space to be later mapped as shared memory views.
///
/// Start address of the region to reserve
/// Size in bytes of the region to reserve
public void ReserveRange(ulong address, ulong size)
{
lock (_mappings)
{
_mappings.Add(new RangeNode(address, address + size, ulong.MaxValue));
}
lock (_protections)
{
_protections.Add(new RangeNode(address, size, MemoryPermission.None));
}
}
///
/// Unreserves a range of memory that has been previously reserved with .
///
/// Start address of the region to unreserve
/// Size in bytes of the region to unreserve
/// Thrown when the Windows API returns an error unreserving the memory
public void UnreserveRange(ulong address, ulong size)
{
ulong endAddress = address + size;
lock (_mappings)
{
RangeNode node = _mappings.GetNode(new RangeNode(address, address + 1UL, default));
RangeNode successorNode;
for (; node != null; node = successorNode)
{
successorNode = node.Successor;
if (IsMapped(node.Value))
{
if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)node.Start, 2))
{
throw new WindowsApiException("UnmapViewOfFile2");
}
}
_mappings.Remove(node);
if (node.End >= endAddress)
{
break;
}
}
}
RemoveProtection(address, size);
}
///
/// Maps a shared memory view on a previously reserved memory region.
///
/// Shared memory that will be the backing storage for the view
/// Offset in the shared memory to map
/// Address to map the view into
/// Size of the view in bytes
/// Memory block that owns the mapping
public void MapView(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size, MemoryBlock owner)
{
ref var partialUnmapLock = ref GetPartialUnmapState().PartialUnmapLock;
partialUnmapLock.AcquireReaderLock();
try
{
UnmapViewInternal(sharedMemory, location, size, owner, updateProtection: false);
MapViewInternal(sharedMemory, srcOffset, location, size);
}
finally
{
partialUnmapLock.ReleaseReaderLock();
}
}
///
/// Maps a shared memory view on a previously reserved memory region.
///
/// Shared memory that will be the backing storage for the view
/// Offset in the shared memory to map
/// Address to map the view into
/// Size of the view in bytes
/// Thrown when the Windows API returns an error mapping the memory
private void MapViewInternal(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size)
{
SplitForMap((ulong)location, (ulong)size, srcOffset);
var ptr = WindowsApi.MapViewOfFile3(
sharedMemory,
WindowsApi.CurrentProcessHandle,
location,
srcOffset,
size,
0x4000,
MemoryProtection.ReadWrite,
IntPtr.Zero,
0);
if (ptr == IntPtr.Zero)
{
throw new WindowsApiException("MapViewOfFile3");
}
UpdateProtection((ulong)location, (ulong)size, MemoryPermission.ReadAndWrite);
}
///
/// Splits a larger placeholder, slicing at the start and end address, for a new memory mapping.
///
/// Address to split
/// Size of the new region
/// Offset in the shared memory that will be mapped
private void SplitForMap(ulong address, ulong size, ulong backingOffset)
{
ulong endAddress = address + size;
var overlaps = new RangeNode[InitialOverlapsSize];
lock (_mappings)
{
int count = _mappings.GetNodes(address, endAddress, ref overlaps);
Debug.Assert(count == 1);
Debug.Assert(!IsMapped(overlaps[0].Value));
var overlap = overlaps[0];
ulong overlapStart = overlap.Start;
ulong overlapEnd = overlap.End;
ulong overlapValue = overlap.Value;
_mappings.Remove(overlap);
bool overlapStartsBefore = overlapStart < address;
bool overlapEndsAfter = overlapEnd > endAddress;
if (overlapStartsBefore && overlapEndsAfter)
{
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)address,
(IntPtr)size,
AllocationType.Release | AllocationType.PreservePlaceholder));
_mappings.Add(new RangeNode(overlapStart, address, overlapValue));
_mappings.Add(new RangeNode(endAddress, overlapEnd, AddBackingOffset(overlapValue, endAddress - overlapStart)));
}
else if (overlapStartsBefore)
{
ulong overlappedSize = overlapEnd - address;
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)address,
(IntPtr)overlappedSize,
AllocationType.Release | AllocationType.PreservePlaceholder));
_mappings.Add(new RangeNode(overlapStart, address, overlapValue));
}
else if (overlapEndsAfter)
{
ulong overlappedSize = endAddress - overlapStart;
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)overlapStart,
(IntPtr)overlappedSize,
AllocationType.Release | AllocationType.PreservePlaceholder));
_mappings.Add(new RangeNode(endAddress, overlapEnd, AddBackingOffset(overlapValue, overlappedSize)));
}
_mappings.Add(new RangeNode(address, endAddress, backingOffset));
}
}
///
/// Unmaps a view that has been previously mapped with .
///
///
/// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be
/// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped.
///
/// Shared memory that the view being unmapped belongs to
/// Address to unmap
/// Size of the region to unmap in bytes
/// Memory block that owns the mapping
public void UnmapView(IntPtr sharedMemory, IntPtr location, IntPtr size, MemoryBlock owner)
{
ref var partialUnmapLock = ref GetPartialUnmapState().PartialUnmapLock;
partialUnmapLock.AcquireReaderLock();
try
{
UnmapViewInternal(sharedMemory, location, size, owner, updateProtection: true);
}
finally
{
partialUnmapLock.ReleaseReaderLock();
}
}
///
/// Unmaps a view that has been previously mapped with .
///
///
/// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be
/// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped.
///
/// Shared memory that the view being unmapped belongs to
/// Address to unmap
/// Size of the region to unmap in bytes
/// Memory block that owns the mapping
/// Indicates if the memory protections should be updated after the unmap
/// Thrown when the Windows API returns an error unmapping or remapping the memory
private void UnmapViewInternal(IntPtr sharedMemory, IntPtr location, IntPtr size, MemoryBlock owner, bool updateProtection)
{
ulong startAddress = (ulong)location;
ulong unmapSize = (ulong)size;
ulong endAddress = startAddress + unmapSize;
var overlaps = new RangeNode[InitialOverlapsSize];
int count;
lock (_mappings)
{
count = _mappings.GetNodes(startAddress, endAddress, ref overlaps);
}
for (int index = 0; index < count; index++)
{
var overlap = overlaps[index];
if (IsMapped(overlap.Value))
{
lock (_mappings)
{
_mappings.Remove(overlap);
_mappings.Add(new RangeNode(overlap.Start, overlap.End, ulong.MaxValue));
}
bool overlapStartsBefore = overlap.Start < startAddress;
bool overlapEndsAfter = overlap.End > endAddress;
if (overlapStartsBefore || overlapEndsAfter)
{
// If the overlap extends beyond the region we are unmapping,
// then we need to re-map the regions that are supposed to remain mapped.
// This is necessary because Windows does not support partial view unmaps.
// That is, you can only fully unmap a view that was previously mapped, you can't just unmap a chunck of it.
ref var partialUnmapState = ref GetPartialUnmapState();
ref var partialUnmapLock = ref partialUnmapState.PartialUnmapLock;
partialUnmapLock.UpgradeToWriterLock();
try
{
partialUnmapState.PartialUnmapsCount++;
if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)overlap.Start, 2))
{
throw new WindowsApiException("UnmapViewOfFile2");
}
if (overlapStartsBefore)
{
ulong remapSize = startAddress - overlap.Start;
MapViewInternal(sharedMemory, overlap.Value, (IntPtr)overlap.Start, (IntPtr)remapSize);
RestoreRangeProtection(overlap.Start, remapSize);
}
if (overlapEndsAfter)
{
ulong overlappedSize = endAddress - overlap.Start;
ulong remapBackingOffset = overlap.Value + overlappedSize;
ulong remapAddress = overlap.Start + overlappedSize;
ulong remapSize = overlap.End - endAddress;
MapViewInternal(sharedMemory, remapBackingOffset, (IntPtr)remapAddress, (IntPtr)remapSize);
RestoreRangeProtection(remapAddress, remapSize);
}
}
finally
{
partialUnmapLock.DowngradeFromWriterLock();
}
}
else if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)overlap.Start, 2))
{
throw new WindowsApiException("UnmapViewOfFile2");
}
}
}
CoalesceForUnmap(startAddress, unmapSize, owner);
if (updateProtection)
{
UpdateProtection(startAddress, unmapSize, MemoryPermission.None);
}
}
///
/// Coalesces adjacent placeholders after unmap.
///
/// Address of the region that was unmapped
/// Size of the region that was unmapped in bytes
/// Memory block that owns the mapping
private void CoalesceForUnmap(ulong address, ulong size, MemoryBlock owner)
{
ulong endAddress = address + size;
ulong blockAddress = (ulong)owner.Pointer;
ulong blockEnd = blockAddress + owner.Size;
int unmappedCount = 0;
lock (_mappings)
{
RangeNode node = _mappings.GetNode(new RangeNode(address, address + 1UL, default));
if (node == null)
{
// Nothing to coalesce if we have no overlaps.
return;
}
RangeNode predecessor = node.Predecessor;
RangeNode successor = null;
for (; node != null; node = successor)
{
successor = node.Successor;
var overlap = node;
if (!IsMapped(overlap.Value))
{
address = Math.Min(address, overlap.Start);
endAddress = Math.Max(endAddress, overlap.End);
_mappings.Remove(overlap);
unmappedCount++;
}
if (node.End >= endAddress)
{
break;
}
}
if (predecessor != null && !IsMapped(predecessor.Value) && predecessor.Start >= blockAddress)
{
address = Math.Min(address, predecessor.Start);
_mappings.Remove(predecessor);
unmappedCount++;
}
if (successor != null && !IsMapped(successor.Value) && successor.End <= blockEnd)
{
endAddress = Math.Max(endAddress, successor.End);
_mappings.Remove(successor);
unmappedCount++;
}
_mappings.Add(new RangeNode(address, endAddress, ulong.MaxValue));
}
if (unmappedCount > 1)
{
size = endAddress - address;
CheckFreeResult(WindowsApi.VirtualFree(
(IntPtr)address,
(IntPtr)size,
AllocationType.Release | AllocationType.CoalescePlaceholders));
}
}
///
/// Reprotects a region of memory that has been mapped.
///
/// Address of the region to reprotect
/// Size of the region to reprotect in bytes
/// New permissions
/// True if the reprotection was successful, false otherwise
public bool ReprotectView(IntPtr address, IntPtr size, MemoryPermission permission)
{
ref var partialUnmapLock = ref GetPartialUnmapState().PartialUnmapLock;
partialUnmapLock.AcquireReaderLock();
try
{
return ReprotectViewInternal(address, size, permission, false);
}
finally
{
partialUnmapLock.ReleaseReaderLock();
}
}
///
/// Reprotects a region of memory that has been mapped.
///
/// Address of the region to reprotect
/// Size of the region to reprotect in bytes
/// New permissions
/// Throw an exception instead of returning an error if the operation fails
/// True if the reprotection was successful or if is true, false otherwise
/// If is true, it is thrown when the Windows API returns an error reprotecting the memory
private bool ReprotectViewInternal(IntPtr address, IntPtr size, MemoryPermission permission, bool throwOnError)
{
ulong reprotectAddress = (ulong)address;
ulong reprotectSize = (ulong)size;
ulong endAddress = reprotectAddress + reprotectSize;
bool success = true;
lock (_mappings)
{
RangeNode node = _mappings.GetNode(new RangeNode(reprotectAddress, reprotectAddress + 1UL, default));
RangeNode successorNode;
for (; node != null; node = successorNode)
{
successorNode = node.Successor;
var overlap = node;
ulong mappedAddress = overlap.Start;
ulong mappedSize = overlap.End - overlap.Start;
if (mappedAddress < reprotectAddress)
{
ulong delta = reprotectAddress - mappedAddress;
mappedAddress = reprotectAddress;
mappedSize -= delta;
}
ulong mappedEndAddress = mappedAddress + mappedSize;
if (mappedEndAddress > endAddress)
{
ulong delta = mappedEndAddress - endAddress;
mappedSize -= delta;
}
if (!WindowsApi.VirtualProtect((IntPtr)mappedAddress, (IntPtr)mappedSize, WindowsApi.GetProtection(permission), out _))
{
if (throwOnError)
{
throw new WindowsApiException("VirtualProtect");
}
success = false;
}
if (node.End >= endAddress)
{
break;
}
}
}
UpdateProtection(reprotectAddress, reprotectSize, permission);
return success;
}
///
/// Checks the result of a VirtualFree operation, throwing if needed.
///
/// Operation result
/// Thrown if is false
private static void CheckFreeResult(bool success)
{
if (!success)
{
throw new WindowsApiException("VirtualFree");
}
}
///
/// Adds an offset to a backing offset. This will do nothing if the backing offset is the special "unmapped" value.
///
/// Backing offset
/// Offset to be added
/// Added offset or just if the region is unmapped
private static ulong AddBackingOffset(ulong backingOffset, ulong offset)
{
if (backingOffset == ulong.MaxValue)
{
return backingOffset;
}
return backingOffset + offset;
}
///
/// Checks if a region is unmapped.
///
/// Backing offset to check
/// True if the backing offset is the special "unmapped" value, false otherwise
private static bool IsMapped(ulong backingOffset)
{
return backingOffset != ulong.MaxValue;
}
///
/// Adds a protection to the list of protections.
///
/// Address of the protected region
/// Size of the protected region in bytes
/// Memory permissions of the region
private void UpdateProtection(ulong address, ulong size, MemoryPermission permission)
{
ulong endAddress = address + size;
lock (_protections)
{
RangeNode node = _protections.GetNode(new RangeNode(address, address + 1UL, default));
if (node != null &&
node.Start <= address &&
node.End >= endAddress &&
node.Value == permission)
{
return;
}
RangeNode successorNode;
ulong startAddress = address;
for (; node != null; node = successorNode)
{
successorNode = node.Successor;
var protection = node;
ulong protAddress = protection.Start;
ulong protEndAddress = protection.End;
MemoryPermission protPermission = protection.Value;
_protections.Remove(protection);
if (protection.Value == permission)
{
if (startAddress > protAddress)
{
startAddress = protAddress;
}
if (endAddress < protEndAddress)
{
endAddress = protEndAddress;
}
}
else
{
if (startAddress > protAddress)
{
_protections.Add(new RangeNode(protAddress, startAddress, protPermission));
}
if (endAddress < protEndAddress)
{
_protections.Add(new RangeNode(endAddress, protEndAddress, protPermission));
}
}
if (node.End >= endAddress)
{
break;
}
}
_protections.Add(new RangeNode(startAddress, endAddress, permission));
}
}
///
/// Removes protection from the list of protections.
///
/// Address of the protected region
/// Size of the protected region in bytes
private void RemoveProtection(ulong address, ulong size)
{
ulong endAddress = address + size;
lock (_protections)
{
RangeNode node = _protections.GetNode(new RangeNode(address, address + 1UL, default));
RangeNode successorNode;
for (; node != null; node = successorNode)
{
successorNode = node.Successor;
var protection = node;
ulong protAddress = protection.Start;
ulong protEndAddress = protection.End;
MemoryPermission protPermission = protection.Value;
_protections.Remove(protection);
if (address > protAddress)
{
_protections.Add(new RangeNode(protAddress, address, protPermission));
}
if (endAddress < protEndAddress)
{
_protections.Add(new RangeNode(endAddress, protEndAddress, protPermission));
}
if (node.End >= endAddress)
{
break;
}
}
}
}
///
/// Restores the protection of a given memory region that was remapped, using the protections list.
///
/// Address of the remapped region
/// Size of the remapped region in bytes
private void RestoreRangeProtection(ulong address, ulong size)
{
ulong endAddress = address + size;
var overlaps = new RangeNode[InitialOverlapsSize];
int count;
lock (_protections)
{
count = _protections.GetNodes(address, endAddress, ref overlaps);
}
ulong startAddress = address;
for (int index = 0; index < count; index++)
{
var protection = overlaps[index];
// If protection is R/W we don't need to reprotect as views are initially mapped as R/W.
if (protection.Value == MemoryPermission.ReadAndWrite)
{
continue;
}
ulong protAddress = protection.Start;
ulong protEndAddress = protection.End;
if (protAddress < address)
{
protAddress = address;
}
if (protEndAddress > endAddress)
{
protEndAddress = endAddress;
}
ReprotectViewInternal((IntPtr)protAddress, (IntPtr)(protEndAddress - protAddress), protection.Value, true);
}
}
}
}