using Ryujinx.Common.Collections; using Ryujinx.Common.Memory.PartialUnmaps; using System; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.Versioning; using System.Threading; namespace Ryujinx.Memory.WindowsShared { /// /// Windows memory placeholder manager. /// [SupportedOSPlatform("windows")] class PlaceholderManager { private const int InitialOverlapsSize = 10; private readonly MappingTree _mappings; private readonly MappingTree _protections; private readonly IntPtr _partialUnmapStatePtr; private readonly Thread _partialUnmapTrimThread; /// /// Creates a new instance of the Windows memory placeholder manager. /// public PlaceholderManager() { _mappings = new MappingTree(); _protections = new MappingTree(); _partialUnmapStatePtr = PartialUnmapState.GlobalState; _partialUnmapTrimThread = new Thread(TrimThreadLocalMapLoop); _partialUnmapTrimThread.Name = "CPU.PartialUnmapTrimThread"; _partialUnmapTrimThread.IsBackground = true; _partialUnmapTrimThread.Start(); } /// /// Gets a reference to the partial unmap state struct. /// /// A reference to the partial unmap state struct private unsafe ref PartialUnmapState GetPartialUnmapState() { return ref Unsafe.AsRef((void*)_partialUnmapStatePtr); } /// /// Trims inactive threads from the partial unmap state's thread mapping every few seconds. /// Should be run in a Background thread so that it doesn't stop the program from closing. /// private void TrimThreadLocalMapLoop() { while (true) { Thread.Sleep(2000); GetPartialUnmapState().TrimThreads(); } } /// /// Reserves a range of the address space to be later mapped as shared memory views. /// /// Start address of the region to reserve /// Size in bytes of the region to reserve public void ReserveRange(ulong address, ulong size) { lock (_mappings) { _mappings.Add(new RangeNode(address, address + size, ulong.MaxValue)); } lock (_protections) { _protections.Add(new RangeNode(address, size, MemoryPermission.None)); } } /// /// Unreserves a range of memory that has been previously reserved with . /// /// Start address of the region to unreserve /// Size in bytes of the region to unreserve /// Thrown when the Windows API returns an error unreserving the memory public void UnreserveRange(ulong address, ulong size) { ulong endAddress = address + size; lock (_mappings) { RangeNode node = _mappings.GetNodeByKey(address); RangeNode successorNode; for (; node != null; node = successorNode) { successorNode = node.Successor; if (IsMapped(node.Value)) { if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)node.Start, 2)) { throw new WindowsApiException("UnmapViewOfFile2"); } } _mappings.Remove(node); if (node.End >= endAddress) { break; } } } RemoveProtection(address, size); } /// /// Maps a shared memory view on a previously reserved memory region. /// /// Shared memory that will be the backing storage for the view /// Offset in the shared memory to map /// Address to map the view into /// Size of the view in bytes /// Memory block that owns the mapping public void MapView(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size, MemoryBlock owner) { ref var partialUnmapLock = ref GetPartialUnmapState().PartialUnmapLock; partialUnmapLock.AcquireReaderLock(); try { UnmapViewInternal(sharedMemory, location, size, owner, updateProtection: false); MapViewInternal(sharedMemory, srcOffset, location, size); } finally { partialUnmapLock.ReleaseReaderLock(); } } /// /// Maps a shared memory view on a previously reserved memory region. /// /// Shared memory that will be the backing storage for the view /// Offset in the shared memory to map /// Address to map the view into /// Size of the view in bytes /// Thrown when the Windows API returns an error mapping the memory private void MapViewInternal(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size) { SplitForMap((ulong)location, (ulong)size, srcOffset); var ptr = WindowsApi.MapViewOfFile3( sharedMemory, WindowsApi.CurrentProcessHandle, location, srcOffset, size, 0x4000, MemoryProtection.ReadWrite, IntPtr.Zero, 0); if (ptr == IntPtr.Zero) { throw new WindowsApiException("MapViewOfFile3"); } UpdateProtection((ulong)location, (ulong)size, MemoryPermission.ReadAndWrite); } /// /// Splits a larger placeholder, slicing at the start and end address, for a new memory mapping. /// /// Address to split /// Size of the new region /// Offset in the shared memory that will be mapped private void SplitForMap(ulong address, ulong size, ulong backingOffset) { ulong endAddress = address + size; var overlaps = new RangeNode[InitialOverlapsSize]; lock (_mappings) { int count = _mappings.GetNodes(address, endAddress, ref overlaps); Debug.Assert(count == 1); Debug.Assert(!IsMapped(overlaps[0].Value)); var overlap = overlaps[0]; ulong overlapStart = overlap.Start; ulong overlapEnd = overlap.End; ulong overlapValue = overlap.Value; _mappings.Remove(overlap); bool overlapStartsBefore = overlapStart < address; bool overlapEndsAfter = overlapEnd > endAddress; if (overlapStartsBefore && overlapEndsAfter) { CheckFreeResult(WindowsApi.VirtualFree( (IntPtr)address, (IntPtr)size, AllocationType.Release | AllocationType.PreservePlaceholder)); _mappings.Add(new RangeNode(overlapStart, address, overlapValue)); _mappings.Add(new RangeNode(endAddress, overlapEnd, AddBackingOffset(overlapValue, endAddress - overlapStart))); } else if (overlapStartsBefore) { ulong overlappedSize = overlapEnd - address; CheckFreeResult(WindowsApi.VirtualFree( (IntPtr)address, (IntPtr)overlappedSize, AllocationType.Release | AllocationType.PreservePlaceholder)); _mappings.Add(new RangeNode(overlapStart, address, overlapValue)); } else if (overlapEndsAfter) { ulong overlappedSize = endAddress - overlapStart; CheckFreeResult(WindowsApi.VirtualFree( (IntPtr)overlapStart, (IntPtr)overlappedSize, AllocationType.Release | AllocationType.PreservePlaceholder)); _mappings.Add(new RangeNode(endAddress, overlapEnd, AddBackingOffset(overlapValue, overlappedSize))); } _mappings.Add(new RangeNode(address, endAddress, backingOffset)); } } /// /// Unmaps a view that has been previously mapped with . /// /// /// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be /// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped. /// /// Shared memory that the view being unmapped belongs to /// Address to unmap /// Size of the region to unmap in bytes /// Memory block that owns the mapping public void UnmapView(IntPtr sharedMemory, IntPtr location, IntPtr size, MemoryBlock owner) { ref var partialUnmapLock = ref GetPartialUnmapState().PartialUnmapLock; partialUnmapLock.AcquireReaderLock(); try { UnmapViewInternal(sharedMemory, location, size, owner, updateProtection: true); } finally { partialUnmapLock.ReleaseReaderLock(); } } /// /// Unmaps a view that has been previously mapped with . /// /// /// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be /// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped. /// /// Shared memory that the view being unmapped belongs to /// Address to unmap /// Size of the region to unmap in bytes /// Memory block that owns the mapping /// Indicates if the memory protections should be updated after the unmap /// Thrown when the Windows API returns an error unmapping or remapping the memory private void UnmapViewInternal(IntPtr sharedMemory, IntPtr location, IntPtr size, MemoryBlock owner, bool updateProtection) { ulong startAddress = (ulong)location; ulong unmapSize = (ulong)size; ulong endAddress = startAddress + unmapSize; var overlaps = new RangeNode[InitialOverlapsSize]; int count; lock (_mappings) { count = _mappings.GetNodes(startAddress, endAddress, ref overlaps); } for (int index = 0; index < count; index++) { var overlap = overlaps[index]; if (IsMapped(overlap.Value)) { lock (_mappings) { _mappings.Remove(overlap); _mappings.Add(new RangeNode(overlap.Start, overlap.End, ulong.MaxValue)); } bool overlapStartsBefore = overlap.Start < startAddress; bool overlapEndsAfter = overlap.End > endAddress; if (overlapStartsBefore || overlapEndsAfter) { // If the overlap extends beyond the region we are unmapping, // then we need to re-map the regions that are supposed to remain mapped. // This is necessary because Windows does not support partial view unmaps. // That is, you can only fully unmap a view that was previously mapped, you can't just unmap a chunck of it. ref var partialUnmapState = ref GetPartialUnmapState(); ref var partialUnmapLock = ref partialUnmapState.PartialUnmapLock; partialUnmapLock.UpgradeToWriterLock(); try { partialUnmapState.PartialUnmapsCount++; if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)overlap.Start, 2)) { throw new WindowsApiException("UnmapViewOfFile2"); } if (overlapStartsBefore) { ulong remapSize = startAddress - overlap.Start; MapViewInternal(sharedMemory, overlap.Value, (IntPtr)overlap.Start, (IntPtr)remapSize); RestoreRangeProtection(overlap.Start, remapSize); } if (overlapEndsAfter) { ulong overlappedSize = endAddress - overlap.Start; ulong remapBackingOffset = overlap.Value + overlappedSize; ulong remapAddress = overlap.Start + overlappedSize; ulong remapSize = overlap.End - endAddress; MapViewInternal(sharedMemory, remapBackingOffset, (IntPtr)remapAddress, (IntPtr)remapSize); RestoreRangeProtection(remapAddress, remapSize); } } finally { partialUnmapLock.DowngradeFromWriterLock(); } } else if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)overlap.Start, 2)) { throw new WindowsApiException("UnmapViewOfFile2"); } } } CoalesceForUnmap(startAddress, unmapSize, owner); if (updateProtection) { UpdateProtection(startAddress, unmapSize, MemoryPermission.None); } } /// /// Coalesces adjacent placeholders after unmap. /// /// Address of the region that was unmapped /// Size of the region that was unmapped in bytes /// Memory block that owns the mapping private void CoalesceForUnmap(ulong address, ulong size, MemoryBlock owner) { ulong endAddress = address + size; ulong blockAddress = (ulong)owner.Pointer; ulong blockEnd = blockAddress + owner.Size; int unmappedCount = 0; lock (_mappings) { RangeNode node = _mappings.GetNodeByKey(address); if (node == null) { // Nothing to coalesce if we have no overlaps. return; } RangeNode predecessor = node.Predecessor; RangeNode successor = null; for (; node != null; node = successor) { successor = node.Successor; var overlap = node; if (!IsMapped(overlap.Value)) { address = Math.Min(address, overlap.Start); endAddress = Math.Max(endAddress, overlap.End); _mappings.Remove(overlap); unmappedCount++; } if (node.End >= endAddress) { break; } } if (predecessor != null && !IsMapped(predecessor.Value) && predecessor.Start >= blockAddress) { address = Math.Min(address, predecessor.Start); _mappings.Remove(predecessor); unmappedCount++; } if (successor != null && !IsMapped(successor.Value) && successor.End <= blockEnd) { endAddress = Math.Max(endAddress, successor.End); _mappings.Remove(successor); unmappedCount++; } _mappings.Add(new RangeNode(address, endAddress, ulong.MaxValue)); } if (unmappedCount > 1) { size = endAddress - address; CheckFreeResult(WindowsApi.VirtualFree( (IntPtr)address, (IntPtr)size, AllocationType.Release | AllocationType.CoalescePlaceholders)); } } /// /// Reprotects a region of memory that has been mapped. /// /// Address of the region to reprotect /// Size of the region to reprotect in bytes /// New permissions /// True if the reprotection was successful, false otherwise public bool ReprotectView(IntPtr address, IntPtr size, MemoryPermission permission) { ref var partialUnmapLock = ref GetPartialUnmapState().PartialUnmapLock; partialUnmapLock.AcquireReaderLock(); try { return ReprotectViewInternal(address, size, permission, false); } finally { partialUnmapLock.ReleaseReaderLock(); } } /// /// Reprotects a region of memory that has been mapped. /// /// Address of the region to reprotect /// Size of the region to reprotect in bytes /// New permissions /// Throw an exception instead of returning an error if the operation fails /// True if the reprotection was successful or if is true, false otherwise /// If is true, it is thrown when the Windows API returns an error reprotecting the memory private bool ReprotectViewInternal(IntPtr address, IntPtr size, MemoryPermission permission, bool throwOnError) { ulong reprotectAddress = (ulong)address; ulong reprotectSize = (ulong)size; ulong endAddress = reprotectAddress + reprotectSize; bool success = true; lock (_mappings) { RangeNode node = _mappings.GetNodeByKey(reprotectAddress); RangeNode successorNode; for (; node != null; node = successorNode) { successorNode = node.Successor; var overlap = node; ulong mappedAddress = overlap.Start; ulong mappedSize = overlap.End - overlap.Start; if (mappedAddress < reprotectAddress) { ulong delta = reprotectAddress - mappedAddress; mappedAddress = reprotectAddress; mappedSize -= delta; } ulong mappedEndAddress = mappedAddress + mappedSize; if (mappedEndAddress > endAddress) { ulong delta = mappedEndAddress - endAddress; mappedSize -= delta; } if (!WindowsApi.VirtualProtect((IntPtr)mappedAddress, (IntPtr)mappedSize, WindowsApi.GetProtection(permission), out _)) { if (throwOnError) { throw new WindowsApiException("VirtualProtect"); } success = false; } if (node.End >= endAddress) { break; } } } UpdateProtection(reprotectAddress, reprotectSize, permission); return success; } /// /// Checks the result of a VirtualFree operation, throwing if needed. /// /// Operation result /// Thrown if is false private static void CheckFreeResult(bool success) { if (!success) { throw new WindowsApiException("VirtualFree"); } } /// /// Adds an offset to a backing offset. This will do nothing if the backing offset is the special "unmapped" value. /// /// Backing offset /// Offset to be added /// Added offset or just if the region is unmapped private static ulong AddBackingOffset(ulong backingOffset, ulong offset) { if (backingOffset == ulong.MaxValue) { return backingOffset; } return backingOffset + offset; } /// /// Checks if a region is unmapped. /// /// Backing offset to check /// True if the backing offset is the special "unmapped" value, false otherwise private static bool IsMapped(ulong backingOffset) { return backingOffset != ulong.MaxValue; } /// /// Adds a protection to the list of protections. /// /// Address of the protected region /// Size of the protected region in bytes /// Memory permissions of the region private void UpdateProtection(ulong address, ulong size, MemoryPermission permission) { ulong endAddress = address + size; lock (_protections) { RangeNode node = _protections.GetNodeByKey(address); if (node != null && node.Start <= address && node.End >= endAddress && node.Value == permission) { return; } RangeNode successorNode; ulong startAddress = address; for (; node != null; node = successorNode) { successorNode = node.Successor; var protection = node; ulong protAddress = protection.Start; ulong protEndAddress = protection.End; MemoryPermission protPermission = protection.Value; _protections.Remove(protection); if (protection.Value == permission) { if (startAddress > protAddress) { startAddress = protAddress; } if (endAddress < protEndAddress) { endAddress = protEndAddress; } } else { if (startAddress > protAddress) { _protections.Add(new RangeNode(protAddress, startAddress, protPermission)); } if (endAddress < protEndAddress) { _protections.Add(new RangeNode(endAddress, protEndAddress, protPermission)); } } if (node.End >= endAddress) { break; } } _protections.Add(new RangeNode(startAddress, endAddress, permission)); } } /// /// Removes protection from the list of protections. /// /// Address of the protected region /// Size of the protected region in bytes private void RemoveProtection(ulong address, ulong size) { ulong endAddress = address + size; lock (_protections) { RangeNode node = _protections.GetNodeByKey(address); RangeNode successorNode; for (; node != null; node = successorNode) { successorNode = node.Successor; var protection = node; ulong protAddress = protection.Start; ulong protEndAddress = protection.End; MemoryPermission protPermission = protection.Value; _protections.Remove(protection); if (address > protAddress) { _protections.Add(new RangeNode(protAddress, address, protPermission)); } if (endAddress < protEndAddress) { _protections.Add(new RangeNode(endAddress, protEndAddress, protPermission)); } if (node.End >= endAddress) { break; } } } } /// /// Restores the protection of a given memory region that was remapped, using the protections list. /// /// Address of the remapped region /// Size of the remapped region in bytes private void RestoreRangeProtection(ulong address, ulong size) { ulong endAddress = address + size; var overlaps = new RangeNode[InitialOverlapsSize]; int count; lock (_protections) { count = _protections.GetNodes(address, endAddress, ref overlaps); } ulong startAddress = address; for (int index = 0; index < count; index++) { var protection = overlaps[index]; // If protection is R/W we don't need to reprotect as views are initially mapped as R/W. if (protection.Value == MemoryPermission.ReadAndWrite) { continue; } ulong protAddress = protection.Start; ulong protEndAddress = protection.End; if (protAddress < address) { protAddress = address; } if (protEndAddress > endAddress) { protEndAddress = endAddress; } ReprotectViewInternal((IntPtr)protAddress, (IntPtr)(protEndAddress - protAddress), protection.Value, true); } } } }