using Ryujinx.Common.Memory;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Memory;
using Ryujinx.Graphics.Gpu.Shader.DiskCache;
using Ryujinx.Graphics.Shader;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.Gpu.Shader
{
class ShaderSpecializationState
{
private const uint ComsMagic = (byte)'C' | ((byte)'O' << 8) | ((byte)'M' << 16) | ((byte)'S' << 24);
private const uint GfxsMagic = (byte)'G' | ((byte)'F' << 8) | ((byte)'X' << 16) | ((byte)'S' << 24);
private const uint TfbdMagic = (byte)'T' | ((byte)'F' << 8) | ((byte)'B' << 16) | ((byte)'D' << 24);
private const uint TexkMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'K' << 24);
private const uint TexsMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'S' << 24);
///
/// Flags indicating GPU state that is used by the shader.
///
[Flags]
private enum QueriedStateFlags
{
EarlyZForce = 1 << 0,
PrimitiveTopology = 1 << 1,
TessellationMode = 1 << 2,
TransformFeedback = 1 << 3
}
private QueriedStateFlags _queriedState;
private bool _compute;
private byte _constantBufferUsePerStage;
///
/// Compute engine state.
///
public GpuChannelComputeState ComputeState;
///
/// 3D engine state.
///
public GpuChannelGraphicsState GraphicsState;
///
/// Contant buffers bound at the time the shader was compiled, per stage.
///
public Array5 ConstantBufferUse;
///
/// Transform feedback buffers active at the time the shader was compiled.
///
public TransformFeedbackDescriptor[] TransformFeedbackDescriptors;
///
/// Flags indicating texture state that is used by the shader.
///
[Flags]
private enum QueriedTextureStateFlags
{
TextureFormat = 1 << 0,
SamplerType = 1 << 1,
CoordNormalized = 1 << 2
}
///
/// Reference type wrapping a value.
///
private class Box
{
///
/// Wrapped value.
///
public T Value;
}
///
/// State of a texture or image that is accessed by the shader.
///
private struct TextureSpecializationState
{
// New fields should be added to the end of the struct to keep disk shader cache compatibility.
///
/// Flags indicating which state of the texture the shader depends on.
///
public QueriedTextureStateFlags QueriedFlags;
///
/// Encoded texture format value.
///
public uint Format;
///
/// True if the texture format is sRGB, false otherwise.
///
public bool FormatSrgb;
///
/// Texture target.
///
public Image.TextureTarget TextureTarget;
///
/// Indicates if the coordinates used to sample the texture are normalized or not (0.0..1.0 or 0..Width/Height).
///
public bool CoordNormalized;
}
///
/// Texture binding information, used to identify each texture accessed by the shader.
///
private struct TextureKey : IEquatable
{
// New fields should be added to the end of the struct to keep disk shader cache compatibility.
///
/// Shader stage where the texture is used.
///
public readonly int StageIndex;
///
/// Texture handle offset in words on the texture buffer.
///
public readonly int Handle;
///
/// Constant buffer slot of the texture buffer (-1 to use the texture buffer index GPU register).
///
public readonly int CbufSlot;
///
/// Creates a new texture key.
///
/// Shader stage where the texture is used
/// Texture handle offset in words on the texture buffer
/// Constant buffer slot of the texture buffer (-1 to use the texture buffer index GPU register)
public TextureKey(int stageIndex, int handle, int cbufSlot)
{
StageIndex = stageIndex;
Handle = handle;
CbufSlot = cbufSlot;
}
public override bool Equals(object obj)
{
return obj is TextureKey textureKey && Equals(textureKey);
}
public bool Equals(TextureKey other)
{
return StageIndex == other.StageIndex && Handle == other.Handle && CbufSlot == other.CbufSlot;
}
public override int GetHashCode()
{
return HashCode.Combine(StageIndex, Handle, CbufSlot);
}
}
private readonly Dictionary> _textureSpecialization;
private KeyValuePair>[] _allTextures;
private Box[][] _textureByBinding;
private Box[][] _imageByBinding;
///
/// Creates a new instance of the shader specialization state.
///
private ShaderSpecializationState()
{
_textureSpecialization = new Dictionary>();
}
///
/// Creates a new instance of the shader specialization state.
///
/// Current compute engine state
public ShaderSpecializationState(GpuChannelComputeState state) : this()
{
ComputeState = state;
_compute = true;
}
///
/// Creates a new instance of the shader specialization state.
///
/// Current 3D engine state
/// Optional transform feedback buffers in use, if any
public ShaderSpecializationState(GpuChannelGraphicsState state, TransformFeedbackDescriptor[] descriptors) : this()
{
GraphicsState = state;
_compute = false;
if (descriptors != null)
{
TransformFeedbackDescriptors = descriptors;
_queriedState |= QueriedStateFlags.TransformFeedback;
}
}
///
/// Prepare the shader specialization state for quick binding lookups.
///
/// The shader stages
public void Prepare(CachedShaderStage[] stages)
{
_allTextures = _textureSpecialization.ToArray();
_textureByBinding = new Box[stages.Length][];
_imageByBinding = new Box[stages.Length][];
for (int i = 0; i < stages.Length; i++)
{
CachedShaderStage stage = stages[i];
if (stage?.Info != null)
{
var textures = stage.Info.Textures;
var images = stage.Info.Images;
var texBindings = new Box[textures.Count];
var imageBindings = new Box[images.Count];
int stageIndex = Math.Max(i - 1, 0); // Don't count VertexA for looking up spec state. No-Op for compute.
for (int j = 0; j < textures.Count; j++)
{
var texture = textures[j];
texBindings[j] = GetTextureSpecState(stageIndex, texture.HandleIndex, texture.CbufSlot);
}
for (int j = 0; j < images.Count; j++)
{
var image = images[j];
imageBindings[j] = GetTextureSpecState(stageIndex, image.HandleIndex, image.CbufSlot);
}
_textureByBinding[i] = texBindings;
_imageByBinding[i] = imageBindings;
}
}
}
///
/// Indicates that the shader accesses the early Z force state.
///
public void RecordEarlyZForce()
{
_queriedState |= QueriedStateFlags.EarlyZForce;
}
///
/// Indicates that the shader accesses the primitive topology state.
///
public void RecordPrimitiveTopology()
{
_queriedState |= QueriedStateFlags.PrimitiveTopology;
}
///
/// Indicates that the shader accesses the tessellation mode state.
///
public void RecordTessellationMode()
{
_queriedState |= QueriedStateFlags.TessellationMode;
}
///
/// Indicates that the shader accesses the constant buffer use state.
///
/// Shader stage index
/// Mask indicating the constant buffers bound at the time of the shader compilation
public void RecordConstantBufferUse(int stageIndex, uint useMask)
{
ConstantBufferUse[stageIndex] = useMask;
_constantBufferUsePerStage |= (byte)(1 << stageIndex);
}
///
/// Indicates that a given texture is accessed by the shader.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
/// Descriptor of the texture
public void RegisterTexture(int stageIndex, int handle, int cbufSlot, Image.TextureDescriptor descriptor)
{
Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot);
state.Value.Format = descriptor.UnpackFormat();
state.Value.FormatSrgb = descriptor.UnpackSrgb();
state.Value.TextureTarget = descriptor.UnpackTextureTarget();
state.Value.CoordNormalized = descriptor.UnpackTextureCoordNormalized();
}
///
/// Indicates that a given texture is accessed by the shader.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
/// Maxwell texture format value
/// Whenever the texture format is a sRGB format
/// Texture target type
/// Whenever the texture coordinates used on the shader are considered normalized
public void RegisterTexture(
int stageIndex,
int handle,
int cbufSlot,
uint format,
bool formatSrgb,
Image.TextureTarget target,
bool coordNormalized)
{
Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot);
state.Value.Format = format;
state.Value.FormatSrgb = formatSrgb;
state.Value.TextureTarget = target;
state.Value.CoordNormalized = coordNormalized;
}
///
/// Indicates that the format of a given texture was used during the shader translation process.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
public void RecordTextureFormat(int stageIndex, int handle, int cbufSlot)
{
Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot);
state.Value.QueriedFlags |= QueriedTextureStateFlags.TextureFormat;
}
///
/// Indicates that the target of a given texture was used during the shader translation process.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
public void RecordTextureSamplerType(int stageIndex, int handle, int cbufSlot)
{
Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot);
state.Value.QueriedFlags |= QueriedTextureStateFlags.SamplerType;
}
///
/// Indicates that the coordinate normalization state of a given texture was used during the shader translation process.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
public void RecordTextureCoordNormalized(int stageIndex, int handle, int cbufSlot)
{
Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot);
state.Value.QueriedFlags |= QueriedTextureStateFlags.CoordNormalized;
}
///
/// Checks if a given texture was registerd on this specialization state.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
public bool TextureRegistered(int stageIndex, int handle, int cbufSlot)
{
return GetTextureSpecState(stageIndex, handle, cbufSlot) != null;
}
///
/// Gets the recorded format of a given texture.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
public (uint, bool) GetFormat(int stageIndex, int handle, int cbufSlot)
{
TextureSpecializationState state = GetTextureSpecState(stageIndex, handle, cbufSlot).Value;
return (state.Format, state.FormatSrgb);
}
///
/// Gets the recorded target of a given texture.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
public Image.TextureTarget GetTextureTarget(int stageIndex, int handle, int cbufSlot)
{
return GetTextureSpecState(stageIndex, handle, cbufSlot).Value.TextureTarget;
}
///
/// Gets the recorded coordinate normalization state of a given texture.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
public bool GetCoordNormalized(int stageIndex, int handle, int cbufSlot)
{
return GetTextureSpecState(stageIndex, handle, cbufSlot).Value.CoordNormalized;
}
///
/// Gets texture specialization state for a given texture, or create a new one if not present.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
/// Texture specialization state
private Box GetOrCreateTextureSpecState(int stageIndex, int handle, int cbufSlot)
{
TextureKey key = new TextureKey(stageIndex, handle, cbufSlot);
if (!_textureSpecialization.TryGetValue(key, out Box state))
{
_textureSpecialization.Add(key, state = new Box());
}
return state;
}
///
/// Gets texture specialization state for a given texture.
///
/// Shader stage where the texture is used
/// Offset in words of the texture handle on the texture buffer
/// Slot of the texture buffer constant buffer
/// Texture specialization state
private Box GetTextureSpecState(int stageIndex, int handle, int cbufSlot)
{
TextureKey key = new TextureKey(stageIndex, handle, cbufSlot);
if (_textureSpecialization.TryGetValue(key, out Box state))
{
return state;
}
return null;
}
///
/// Checks if the recorded state matches the current GPU 3D engine state.
///
/// GPU channel
/// Texture pool state
/// Graphics state
/// Indicates whether texture descriptors should be checked
/// True if the state matches, false otherwise
public bool MatchesGraphics(GpuChannel channel, GpuChannelPoolState poolState, GpuChannelGraphicsState graphicsState, bool checkTextures)
{
if (graphicsState.ViewportTransformDisable != GraphicsState.ViewportTransformDisable)
{
return false;
}
return Matches(channel, poolState, checkTextures, isCompute: false);
}
///
/// Checks if the recorded state matches the current GPU compute engine state.
///
/// GPU channel
/// Texture pool state
/// Indicates whether texture descriptors should be checked
/// True if the state matches, false otherwise
public bool MatchesCompute(GpuChannel channel, GpuChannelPoolState poolState, bool checkTextures)
{
return Matches(channel, poolState, checkTextures, isCompute: true);
}
///
/// Fetch the constant buffers used for a texture to cache.
///
/// GPU channel
/// Indicates whenever the check is requested by the 3D or compute engine
/// The currently cached texture buffer index
/// The currently cached sampler buffer index
/// The currently cached texture buffer data
/// The currently cached sampler buffer data
/// The currently cached stage
/// The new texture buffer index
/// The new sampler buffer index
/// Stage index of the constant buffer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void UpdateCachedBuffer(
GpuChannel channel,
bool isCompute,
ref int cachedTextureBufferIndex,
ref int cachedSamplerBufferIndex,
ref ReadOnlySpan cachedTextureBuffer,
ref ReadOnlySpan cachedSamplerBuffer,
ref int cachedStageIndex,
int textureBufferIndex,
int samplerBufferIndex,
int stageIndex)
{
bool stageChange = stageIndex != cachedStageIndex;
if (stageChange || textureBufferIndex != cachedTextureBufferIndex)
{
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex);
cachedTextureBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
cachedTextureBufferIndex = textureBufferIndex;
if (samplerBufferIndex == textureBufferIndex)
{
cachedSamplerBuffer = cachedTextureBuffer;
cachedSamplerBufferIndex = samplerBufferIndex;
}
}
if (stageChange || samplerBufferIndex != cachedSamplerBufferIndex)
{
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex);
cachedSamplerBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
cachedSamplerBufferIndex = samplerBufferIndex;
}
cachedStageIndex = stageIndex;
}
///
/// Checks if the recorded state matches the current GPU state.
///
/// GPU channel
/// Texture pool state
/// Indicates whether texture descriptors should be checked
/// Indicates whenever the check is requested by the 3D or compute engine
/// True if the state matches, false otherwise
private bool Matches(GpuChannel channel, GpuChannelPoolState poolState, bool checkTextures, bool isCompute)
{
int constantBufferUsePerStageMask = _constantBufferUsePerStage;
while (constantBufferUsePerStageMask != 0)
{
int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask);
uint useMask = isCompute
? channel.BufferManager.GetComputeUniformBufferUseMask()
: channel.BufferManager.GetGraphicsUniformBufferUseMask(index);
if (ConstantBufferUse[index] != useMask)
{
return false;
}
constantBufferUsePerStageMask &= ~(1 << index);
}
if (checkTextures)
{
TexturePool pool = channel.TextureManager.GetTexturePool(poolState.TexturePoolGpuVa, poolState.TexturePoolMaximumId);
int cachedTextureBufferIndex = -1;
int cachedSamplerBufferIndex = -1;
int cachedStageIndex = -1;
ReadOnlySpan cachedTextureBuffer = Span.Empty;
ReadOnlySpan cachedSamplerBuffer = Span.Empty;
foreach (var kv in _allTextures)
{
TextureKey textureKey = kv.Key;
(int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(textureKey.CbufSlot, poolState.TextureBufferIndex);
UpdateCachedBuffer(channel,
isCompute,
ref cachedTextureBufferIndex,
ref cachedSamplerBufferIndex,
ref cachedTextureBuffer,
ref cachedSamplerBuffer,
ref cachedStageIndex,
textureBufferIndex,
samplerBufferIndex,
textureKey.StageIndex);
int packedId = TextureHandle.ReadPackedId(textureKey.Handle, cachedTextureBuffer, cachedSamplerBuffer);
int textureId = TextureHandle.UnpackTextureId(packedId);
if (pool.IsValidId(textureId))
{
ref readonly Image.TextureDescriptor descriptor = ref pool.GetDescriptorRef(textureId);
if (!MatchesTexture(kv.Value, descriptor))
{
return false;
}
}
}
}
return true;
}
///
/// Checks if the recorded texture state matches the given texture descriptor.
///
/// Texture specialization state
/// Texture descriptor
/// True if the state matches, false otherwise
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool MatchesTexture(Box specializationState, in Image.TextureDescriptor descriptor)
{
if (specializationState != null)
{
if (specializationState.Value.QueriedFlags.HasFlag(QueriedTextureStateFlags.CoordNormalized) &&
specializationState.Value.CoordNormalized != descriptor.UnpackTextureCoordNormalized())
{
return false;
}
}
return true;
}
///
/// Checks if the recorded texture state for a given texture binding matches a texture descriptor.
///
/// The shader stage
/// The texture index
/// Texture descriptor
/// True if the state matches, false otherwise
public bool MatchesTexture(ShaderStage stage, int index, in Image.TextureDescriptor descriptor)
{
Box specializationState = _textureByBinding[(int)stage][index];
return MatchesTexture(specializationState, descriptor);
}
///
/// Checks if the recorded texture state for a given image binding matches a texture descriptor.
///
/// The shader stage
/// The texture index
/// Texture descriptor
/// True if the state matches, false otherwise
public bool MatchesImage(ShaderStage stage, int index, in Image.TextureDescriptor descriptor)
{
Box specializationState = _imageByBinding[(int)stage][index];
return MatchesTexture(specializationState, descriptor);
}
///
/// Reads shader specialization state that has been serialized.
///
/// Data reader
/// Shader specialization state
public static ShaderSpecializationState Read(ref BinarySerializer dataReader)
{
ShaderSpecializationState specState = new ShaderSpecializationState();
dataReader.Read(ref specState._queriedState);
dataReader.Read(ref specState._compute);
if (specState._compute)
{
dataReader.ReadWithMagicAndSize(ref specState.ComputeState, ComsMagic);
}
else
{
dataReader.ReadWithMagicAndSize(ref specState.GraphicsState, GfxsMagic);
}
dataReader.Read(ref specState._constantBufferUsePerStage);
int constantBufferUsePerStageMask = specState._constantBufferUsePerStage;
while (constantBufferUsePerStageMask != 0)
{
int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask);
dataReader.Read(ref specState.ConstantBufferUse[index]);
constantBufferUsePerStageMask &= ~(1 << index);
}
if (specState._queriedState.HasFlag(QueriedStateFlags.TransformFeedback))
{
ushort tfCount = 0;
dataReader.Read(ref tfCount);
specState.TransformFeedbackDescriptors = new TransformFeedbackDescriptor[tfCount];
for (int index = 0; index < tfCount; index++)
{
dataReader.ReadWithMagicAndSize(ref specState.TransformFeedbackDescriptors[index], TfbdMagic);
}
}
ushort count = 0;
dataReader.Read(ref count);
for (int index = 0; index < count; index++)
{
TextureKey textureKey = default;
Box textureState = new Box();
dataReader.ReadWithMagicAndSize(ref textureKey, TexkMagic);
dataReader.ReadWithMagicAndSize(ref textureState.Value, TexsMagic);
specState._textureSpecialization[textureKey] = textureState;
}
return specState;
}
///
/// Serializes the shader specialization state.
///
/// Data writer
public void Write(ref BinarySerializer dataWriter)
{
dataWriter.Write(ref _queriedState);
dataWriter.Write(ref _compute);
if (_compute)
{
dataWriter.WriteWithMagicAndSize(ref ComputeState, ComsMagic);
}
else
{
dataWriter.WriteWithMagicAndSize(ref GraphicsState, GfxsMagic);
}
dataWriter.Write(ref _constantBufferUsePerStage);
int constantBufferUsePerStageMask = _constantBufferUsePerStage;
while (constantBufferUsePerStageMask != 0)
{
int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask);
dataWriter.Write(ref ConstantBufferUse[index]);
constantBufferUsePerStageMask &= ~(1 << index);
}
if (_queriedState.HasFlag(QueriedStateFlags.TransformFeedback))
{
ushort tfCount = (ushort)TransformFeedbackDescriptors.Length;
dataWriter.Write(ref tfCount);
for (int index = 0; index < TransformFeedbackDescriptors.Length; index++)
{
dataWriter.WriteWithMagicAndSize(ref TransformFeedbackDescriptors[index], TfbdMagic);
}
}
ushort count = (ushort)_textureSpecialization.Count;
dataWriter.Write(ref count);
foreach (var kv in _textureSpecialization)
{
var textureKey = kv.Key;
var textureState = kv.Value;
dataWriter.WriteWithMagicAndSize(ref textureKey, TexkMagic);
dataWriter.WriteWithMagicAndSize(ref textureState.Value, TexsMagic);
}
}
}
}