using Ryujinx.Common.Logging; using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Gpu.Engine.Types; using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Gpu.Shader; using Ryujinx.Graphics.Shader; using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Image { /// /// Texture bindings manager. /// class TextureBindingsManager : IDisposable { private const int InitialTextureStateSize = 32; private const int InitialImageStateSize = 8; private readonly GpuContext _context; private readonly bool _isCompute; private SamplerPool _samplerPool; private SamplerIndex _samplerIndex; private ulong _texturePoolAddress; private int _texturePoolMaximumId; private readonly GpuChannel _channel; private readonly TexturePoolCache _texturePoolCache; private TexturePool _cachedTexturePool; private SamplerPool _cachedSamplerPool; private readonly TextureBindingInfo[][] _textureBindings; private readonly TextureBindingInfo[][] _imageBindings; private struct TextureState { public ITexture Texture; public ISampler Sampler; public int TextureHandle; public int SamplerHandle; public int InvalidatedSequence; public Texture CachedTexture; public Sampler CachedSampler; public int ScaleIndex; public TextureUsageFlags UsageFlags; } private TextureState[] _textureState; private TextureState[] _imageState; private int[] _textureBindingsCount; private int[] _imageBindingsCount; private int _texturePoolSequence; private int _samplerPoolSequence; private int _textureBufferIndex; private readonly float[] _scales; private bool _scaleChanged; private int _lastFragmentTotal; /// /// Constructs a new instance of the texture bindings manager. /// /// The GPU context that the texture bindings manager belongs to /// The GPU channel that the texture bindings manager belongs to /// Texture pools cache used to get texture pools from /// Array where the scales for the currently bound textures are stored /// True if the bindings manager is used for the compute engine public TextureBindingsManager(GpuContext context, GpuChannel channel, TexturePoolCache poolCache, float[] scales, bool isCompute) { _context = context; _channel = channel; _texturePoolCache = poolCache; _scales = scales; _isCompute = isCompute; int stages = isCompute ? 1 : Constants.ShaderStages; _textureBindings = new TextureBindingInfo[stages][]; _imageBindings = new TextureBindingInfo[stages][]; _textureState = new TextureState[InitialTextureStateSize]; _imageState = new TextureState[InitialImageStateSize]; _textureBindingsCount = new int[stages]; _imageBindingsCount = new int[stages]; for (int stage = 0; stage < stages; stage++) { _textureBindings[stage] = new TextureBindingInfo[InitialTextureStateSize]; _imageBindings[stage] = new TextureBindingInfo[InitialImageStateSize]; } } /// /// Rents the texture bindings array for a given stage, so that they can be modified. /// /// Shader stage number, or 0 for compute shaders /// The number of bindings needed /// The texture bindings array public TextureBindingInfo[] RentTextureBindings(int stage, int count) { if (count > _textureBindings[stage].Length) { Array.Resize(ref _textureBindings[stage], count); } _textureBindingsCount[stage] = count; return _textureBindings[stage]; } /// /// Rents the image bindings array for a given stage, so that they can be modified. /// /// Shader stage number, or 0 for compute shaders /// The number of bindings needed /// The image bindings array public TextureBindingInfo[] RentImageBindings(int stage, int count) { if (count > _imageBindings[stage].Length) { Array.Resize(ref _imageBindings[stage], count); } _imageBindingsCount[stage] = count; return _imageBindings[stage]; } /// /// Sets the max binding indexes for textures and images. /// /// The maximum texture binding /// The maximum image binding public void SetMaxBindings(int maxTextureBinding, int maxImageBinding) { if (maxTextureBinding >= _textureState.Length) { Array.Resize(ref _textureState, maxTextureBinding + 1); } if (maxImageBinding >= _imageState.Length) { Array.Resize(ref _imageState, maxImageBinding + 1); } } /// /// Sets the textures constant buffer index. /// The constant buffer specified holds the texture handles. /// /// Constant buffer index public void SetTextureBufferIndex(int index) { _textureBufferIndex = index; } /// /// Sets the current texture sampler pool to be used. /// /// Start GPU virtual address of the pool /// Maximum ID of the pool (total count minus one) /// Type of the sampler pool indexing used for bound samplers public void SetSamplerPool(ulong gpuVa, int maximumId, SamplerIndex samplerIndex) { if (gpuVa != 0) { ulong address = _channel.MemoryManager.Translate(gpuVa); if (_samplerPool != null && _samplerPool.Address == address && _samplerPool.MaximumId >= maximumId) { return; } _samplerPool?.Dispose(); _samplerPool = new SamplerPool(_context, _channel.MemoryManager.Physical, address, maximumId); } else { _samplerPool?.Dispose(); _samplerPool = null; } _samplerIndex = samplerIndex; } /// /// Sets the current texture pool to be used. /// /// Start GPU virtual address of the pool /// Maximum ID of the pool (total count minus one) public void SetTexturePool(ulong gpuVa, int maximumId) { if (gpuVa != 0) { ulong address = _channel.MemoryManager.Translate(gpuVa); _texturePoolAddress = address; _texturePoolMaximumId = maximumId; } else { _texturePoolAddress = 0; _texturePoolMaximumId = 0; } } /// /// Gets a texture and a sampler from their respective pools from a texture ID and a sampler ID. /// /// ID of the texture /// ID of the sampler public (Texture, Sampler) GetTextureAndSampler(int textureId, int samplerId) { ulong texturePoolAddress = _texturePoolAddress; TexturePool texturePool = texturePoolAddress != 0 ? _texturePoolCache.FindOrCreate(_channel, texturePoolAddress, _texturePoolMaximumId) : null; return (texturePool.Get(textureId), _samplerPool.Get(samplerId)); } /// /// Updates the texture scale for a given texture or image. /// /// Start GPU virtual address of the pool /// The related texture usage flags /// The texture/image binding index /// The active shader stage /// True if the given texture has become blacklisted, indicating that its host texture may have changed. private bool UpdateScale(Texture texture, TextureUsageFlags usageFlags, int index, ShaderStage stage) { float result = 1f; bool changed = false; if ((usageFlags & TextureUsageFlags.NeedsScaleValue) != 0 && texture != null) { if ((usageFlags & TextureUsageFlags.ResScaleUnsupported) != 0) { changed = texture.ScaleMode != TextureScaleMode.Blacklisted; texture.BlacklistScale(); } else { switch (stage) { case ShaderStage.Fragment: float scale = texture.ScaleFactor; if (scale != 1) { Texture activeTarget = _channel.TextureManager.GetAnyRenderTarget(); if (activeTarget != null && (activeTarget.Info.Width / (float)texture.Info.Width) == (activeTarget.Info.Height / (float)texture.Info.Height)) { // If the texture's size is a multiple of the sampler size, enable interpolation using gl_FragCoord. (helps "invent" new integer values between scaled pixels) result = -scale; break; } } result = scale; break; case ShaderStage.Vertex: int fragmentIndex = (int)ShaderStage.Fragment - 1; index += _textureBindingsCount[fragmentIndex] + _imageBindingsCount[fragmentIndex]; result = texture.ScaleFactor; break; case ShaderStage.Compute: result = texture.ScaleFactor; break; } } } if (result != _scales[index]) { _scaleChanged = true; _scales[index] = result; } return changed; } /// /// Uploads texture and image scales to the backend when they are used. /// private void CommitRenderScale() { // Stage 0 total: Compute or Vertex. int total = _textureBindingsCount[0] + _imageBindingsCount[0]; int fragmentIndex = (int)ShaderStage.Fragment - 1; int fragmentTotal = _isCompute ? 0 : (_textureBindingsCount[fragmentIndex] + _imageBindingsCount[fragmentIndex]); if (total != 0 && fragmentTotal != _lastFragmentTotal) { // Must update scales in the support buffer if: // - Vertex stage has bindings. // - Fragment stage binding count has been updated since last render scale update. _scaleChanged = true; } if (_scaleChanged) { if (!_isCompute) { total += fragmentTotal; // Add the fragment bindings to the total. } _lastFragmentTotal = fragmentTotal; _context.Renderer.Pipeline.UpdateRenderScale(_scales, total, fragmentTotal); _scaleChanged = false; } } /// /// Ensures that the bindings are visible to the host GPU. /// Note: this actually performs the binding using the host graphics API. /// /// Specialization state for the bound shader /// True if all bound textures match the current shader specialiation state, false otherwise public bool CommitBindings(ShaderSpecializationState specState) { ulong texturePoolAddress = _texturePoolAddress; TexturePool texturePool = texturePoolAddress != 0 ? _texturePoolCache.FindOrCreate(_channel, texturePoolAddress, _texturePoolMaximumId) : null; SamplerPool samplerPool = _samplerPool; // Check if the texture pool has been modified since bindings were last committed. // If it wasn't, then it's possible to avoid looking up textures again when the handle remains the same. bool poolModified = _cachedTexturePool != texturePool || _cachedSamplerPool != samplerPool; _cachedTexturePool = texturePool; _cachedSamplerPool = samplerPool; if (texturePool != null) { int texturePoolSequence = texturePool.CheckModified(); if (_texturePoolSequence != texturePoolSequence) { poolModified = true; _texturePoolSequence = texturePoolSequence; } } if (samplerPool != null) { int samplerPoolSequence = samplerPool.CheckModified(); if (_samplerPoolSequence != samplerPoolSequence) { poolModified = true; _samplerPoolSequence = samplerPoolSequence; } } bool specStateMatches = true; if (_isCompute) { specStateMatches &= CommitTextureBindings(texturePool, ShaderStage.Compute, 0, poolModified, specState); specStateMatches &= CommitImageBindings(texturePool, ShaderStage.Compute, 0, poolModified, specState); } else { for (ShaderStage stage = ShaderStage.Vertex; stage <= ShaderStage.Fragment; stage++) { int stageIndex = (int)stage - 1; specStateMatches &= CommitTextureBindings(texturePool, stage, stageIndex, poolModified, specState); specStateMatches &= CommitImageBindings(texturePool, stage, stageIndex, poolModified, specState); } } CommitRenderScale(); return specStateMatches; } /// /// Fetch the constant buffers used for a texture to cache. /// /// Stage index of the constant buffer /// The currently cached texture buffer index /// The currently cached sampler buffer index /// The currently cached texture buffer data /// The currently cached sampler buffer data /// The new texture buffer index /// The new sampler buffer index [MethodImpl(MethodImplOptions.AggressiveInlining)] private void UpdateCachedBuffer( int stageIndex, ref int cachedTextureBufferIndex, ref int cachedSamplerBufferIndex, ref ReadOnlySpan cachedTextureBuffer, ref ReadOnlySpan cachedSamplerBuffer, int textureBufferIndex, int samplerBufferIndex) { if (textureBufferIndex != cachedTextureBufferIndex) { ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex); cachedTextureBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedTextureBufferIndex = textureBufferIndex; if (samplerBufferIndex == textureBufferIndex) { cachedSamplerBuffer = cachedTextureBuffer; cachedSamplerBufferIndex = samplerBufferIndex; } } if (samplerBufferIndex != cachedSamplerBufferIndex) { ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex); cachedSamplerBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedSamplerBufferIndex = samplerBufferIndex; } } /// /// Ensures that the texture bindings are visible to the host GPU. /// Note: this actually performs the binding using the host graphics API. /// /// The current texture pool /// The shader stage using the textures to be bound /// The stage number of the specified shader stageTrue if either the texture or sampler pool was modified, false otherwise /// Specialization state for the bound shader /// True if all bound textures match the current shader specialiation state, false otherwise private bool CommitTextureBindings(TexturePool pool, ShaderStage stage, int stageIndex, bool poolModified, ShaderSpecializationState specState) { int textureCount = _textureBindingsCount[stageIndex]; if (textureCount == 0) { return true; } var samplerPool = _samplerPool; if (pool == null) { Logger.Error?.Print(LogClass.Gpu, $"Shader stage \"{stage}\" uses textures, but texture pool was not set."); return true; } bool specStateMatches = true; int cachedTextureBufferIndex = -1; int cachedSamplerBufferIndex = -1; ReadOnlySpan cachedTextureBuffer = Span.Empty; ReadOnlySpan cachedSamplerBuffer = Span.Empty; for (int index = 0; index < textureCount; index++) { TextureBindingInfo bindingInfo = _textureBindings[stageIndex][index]; TextureUsageFlags usageFlags = bindingInfo.Flags; (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(bindingInfo.CbufSlot, _textureBufferIndex); UpdateCachedBuffer(stageIndex, ref cachedTextureBufferIndex, ref cachedSamplerBufferIndex, ref cachedTextureBuffer, ref cachedSamplerBuffer, textureBufferIndex, samplerBufferIndex); int packedId = TextureHandle.ReadPackedId(bindingInfo.Handle, cachedTextureBuffer, cachedSamplerBuffer); int textureId = TextureHandle.UnpackTextureId(packedId); int samplerId; if (_samplerIndex == SamplerIndex.ViaHeaderIndex) { samplerId = textureId; } else { samplerId = TextureHandle.UnpackSamplerId(packedId); } ref TextureState state = ref _textureState[bindingInfo.Binding]; if (!poolModified && state.TextureHandle == textureId && state.SamplerHandle == samplerId && state.CachedTexture != null && state.CachedTexture.InvalidatedSequence == state.InvalidatedSequence && state.CachedSampler?.IsDisposed != true) { // The texture is already bound. state.CachedTexture.SynchronizeMemory(); if ((state.ScaleIndex != index || state.UsageFlags != usageFlags) && UpdateScale(state.CachedTexture, usageFlags, index, stage)) { ITexture hostTextureRebind = state.CachedTexture.GetTargetTexture(bindingInfo.Target); state.Texture = hostTextureRebind; state.ScaleIndex = index; state.UsageFlags = usageFlags; _context.Renderer.Pipeline.SetTexture(bindingInfo.Binding, hostTextureRebind); } continue; } state.TextureHandle = textureId; state.SamplerHandle = samplerId; ref readonly TextureDescriptor descriptor = ref pool.GetForBinding(textureId, out Texture texture); specStateMatches &= specState.MatchesTexture(stage, index, descriptor); ITexture hostTexture = texture?.GetTargetTexture(bindingInfo.Target); if (hostTexture != null && texture.Target == Target.TextureBuffer) { // Ensure that the buffer texture is using the correct buffer as storage. // Buffers are frequently re-created to accomodate larger data, so we need to re-bind // to ensure we're not using a old buffer that was already deleted. _channel.BufferManager.SetBufferTextureStorage(hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, bindingInfo.Format, false); } else { if (state.Texture != hostTexture) { if (UpdateScale(texture, usageFlags, index, stage)) { hostTexture = texture?.GetTargetTexture(bindingInfo.Target); } state.Texture = hostTexture; state.ScaleIndex = index; state.UsageFlags = usageFlags; _context.Renderer.Pipeline.SetTexture(bindingInfo.Binding, hostTexture); } Sampler sampler = samplerPool?.Get(samplerId); state.CachedSampler = sampler; ISampler hostSampler = sampler?.GetHostSampler(texture); if (state.Sampler != hostSampler) { state.Sampler = hostSampler; _context.Renderer.Pipeline.SetSampler(bindingInfo.Binding, hostSampler); } state.CachedTexture = texture; state.InvalidatedSequence = texture?.InvalidatedSequence ?? 0; } } return specStateMatches; } /// /// Ensures that the image bindings are visible to the host GPU. /// Note: this actually performs the binding using the host graphics API. /// /// The current texture pool /// The shader stage using the textures to be bound /// The stage number of the specified shader stage /// True if either the texture or sampler pool was modified, false otherwise /// Specialization state for the bound shader /// True if all bound images match the current shader specialiation state, false otherwise private bool CommitImageBindings(TexturePool pool, ShaderStage stage, int stageIndex, bool poolModified, ShaderSpecializationState specState) { int imageCount = _imageBindingsCount[stageIndex]; if (imageCount == 0) { return true; } if (pool == null) { Logger.Error?.Print(LogClass.Gpu, $"Shader stage \"{stage}\" uses images, but texture pool was not set."); return true; } // Scales for images appear after the texture ones. int baseScaleIndex = _textureBindingsCount[stageIndex]; int cachedTextureBufferIndex = -1; int cachedSamplerBufferIndex = -1; ReadOnlySpan cachedTextureBuffer = Span.Empty; ReadOnlySpan cachedSamplerBuffer = Span.Empty; bool specStateMatches = true; for (int index = 0; index < imageCount; index++) { TextureBindingInfo bindingInfo = _imageBindings[stageIndex][index]; TextureUsageFlags usageFlags = bindingInfo.Flags; int scaleIndex = baseScaleIndex + index; (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(bindingInfo.CbufSlot, _textureBufferIndex); UpdateCachedBuffer(stageIndex, ref cachedTextureBufferIndex, ref cachedSamplerBufferIndex, ref cachedTextureBuffer, ref cachedSamplerBuffer, textureBufferIndex, samplerBufferIndex); int packedId = TextureHandle.ReadPackedId(bindingInfo.Handle, cachedTextureBuffer, cachedSamplerBuffer); int textureId = TextureHandle.UnpackTextureId(packedId); ref TextureState state = ref _imageState[bindingInfo.Binding]; bool isStore = bindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore); if (!poolModified && state.TextureHandle == textureId && state.CachedTexture != null && state.CachedTexture.InvalidatedSequence == state.InvalidatedSequence) { Texture cachedTexture = state.CachedTexture; // The texture is already bound. cachedTexture.SynchronizeMemory(); if (isStore) { cachedTexture?.SignalModified(); } if ((state.ScaleIndex != index || state.UsageFlags != usageFlags) && UpdateScale(state.CachedTexture, usageFlags, scaleIndex, stage)) { ITexture hostTextureRebind = state.CachedTexture.GetTargetTexture(bindingInfo.Target); Format format = bindingInfo.Format == 0 ? cachedTexture.Format : bindingInfo.Format; state.Texture = hostTextureRebind; state.ScaleIndex = scaleIndex; state.UsageFlags = usageFlags; _context.Renderer.Pipeline.SetImage(bindingInfo.Binding, hostTextureRebind, format); } continue; } state.TextureHandle = textureId; ref readonly TextureDescriptor descriptor = ref pool.GetForBinding(textureId, out Texture texture); specStateMatches &= specState.MatchesImage(stage, index, descriptor); ITexture hostTexture = texture?.GetTargetTexture(bindingInfo.Target); if (hostTexture != null && texture.Target == Target.TextureBuffer) { // Ensure that the buffer texture is using the correct buffer as storage. // Buffers are frequently re-created to accomodate larger data, so we need to re-bind // to ensure we're not using a old buffer that was already deleted. Format format = bindingInfo.Format; if (format == 0 && texture != null) { format = texture.Format; } _channel.BufferManager.SetBufferTextureStorage(hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, format, true); } else { if (isStore) { texture?.SignalModified(); } if (state.Texture != hostTexture) { if (UpdateScale(texture, usageFlags, scaleIndex, stage)) { hostTexture = texture?.GetTargetTexture(bindingInfo.Target); } state.Texture = hostTexture; state.ScaleIndex = scaleIndex; state.UsageFlags = usageFlags; Format format = bindingInfo.Format; if (format == 0 && texture != null) { format = texture.Format; } _context.Renderer.Pipeline.SetImage(bindingInfo.Binding, hostTexture, format); } state.CachedTexture = texture; state.InvalidatedSequence = texture?.InvalidatedSequence ?? 0; } } return specStateMatches; } /// /// Gets the texture descriptor for a given texture handle. /// /// GPU virtual address of the texture pool /// Index of the constant buffer with texture handles /// Maximum ID of the texture pool /// The stage number where the texture is bound /// The texture handle /// The texture handle's constant buffer slot /// The texture descriptor for the specified texture public TextureDescriptor GetTextureDescriptor( ulong poolGpuVa, int bufferIndex, int maximumId, int stageIndex, int handle, int cbufSlot) { (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(cbufSlot, bufferIndex); int packedId = ReadPackedId(stageIndex, handle, textureBufferIndex, samplerBufferIndex); int textureId = TextureHandle.UnpackTextureId(packedId); ulong poolAddress = _channel.MemoryManager.Translate(poolGpuVa); TexturePool texturePool = _texturePoolCache.FindOrCreate(_channel, poolAddress, maximumId); TextureDescriptor descriptor; if (texturePool.IsValidId(textureId)) { descriptor = texturePool.GetDescriptor(textureId); } else { // If the ID is not valid, we just return a default descriptor with the most common state. // Since this is used for shader specialization, doing so might avoid the need for recompilations. descriptor = new TextureDescriptor(); descriptor.Word4 |= (uint)TextureTarget.Texture2D << 23; descriptor.Word5 |= 1u << 31; // Coords normalized. } return descriptor; } /// /// Reads a packed texture and sampler ID (basically, the real texture handle) /// from the texture constant buffer. /// /// The number of the shader stage where the texture is bound /// A word offset of the handle on the buffer (the "fake" shader handle) /// Index of the constant buffer holding the texture handles /// Index of the constant buffer holding the sampler handles /// The packed texture and sampler ID (the real texture handle) [MethodImpl(MethodImplOptions.AggressiveInlining)] private int ReadPackedId(int stageIndex, int wordOffset, int textureBufferIndex, int samplerBufferIndex) { (int textureWordOffset, int samplerWordOffset, TextureHandleType handleType) = TextureHandle.UnpackOffsets(wordOffset); ulong textureBufferAddress = _isCompute ? _channel.BufferManager.GetComputeUniformBufferAddress(textureBufferIndex) : _channel.BufferManager.GetGraphicsUniformBufferAddress(stageIndex, textureBufferIndex); int handle = _channel.MemoryManager.Physical.Read(textureBufferAddress + (uint)textureWordOffset * 4); // The "wordOffset" (which is really the immediate value used on texture instructions on the shader) // is a 13-bit value. However, in order to also support separate samplers and textures (which uses // bindless textures on the shader), we extend it with another value on the higher 16 bits with // another offset for the sampler. // The shader translator has code to detect separate texture and sampler uses with a bindless texture, // turn that into a regular texture access and produce those special handles with values on the higher 16 bits. if (handleType != TextureHandleType.CombinedSampler) { int samplerHandle; if (handleType != TextureHandleType.SeparateConstantSamplerHandle) { ulong samplerBufferAddress = _isCompute ? _channel.BufferManager.GetComputeUniformBufferAddress(samplerBufferIndex) : _channel.BufferManager.GetGraphicsUniformBufferAddress(stageIndex, samplerBufferIndex); samplerHandle = _channel.MemoryManager.Physical.Read(samplerBufferAddress + (uint)samplerWordOffset * 4); } else { samplerHandle = samplerWordOffset; } if (handleType == TextureHandleType.SeparateSamplerId || handleType == TextureHandleType.SeparateConstantSamplerHandle) { samplerHandle <<= 20; } handle |= samplerHandle; } return handle; } /// /// Force all bound textures and images to be rebound the next time CommitBindings is called. /// public void Rebind() { Array.Clear(_textureState); Array.Clear(_imageState); } /// /// Disposes all textures and samplers in the cache. /// public void Dispose() { _samplerPool?.Dispose(); } } }