Make StencilOp, FrontFace and CullMode Dynamic States. Also prevent Vertex Input Binding Stride dynamic state from enabling with MoltenVK.

src/Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -84,6 +84,8 @@ namespace Ryujinx.Graphics.Vulkan
         private bool _tfEnabled;
         private bool _tfActive;
 
+        private bool _supportExtDynamic;
+
         private readonly PipelineColorBlendAttachmentState[] _storedBlend;
 
         private ulong _drawCountSinceBarrier;
@@ -122,6 +124,8 @@ namespace Ryujinx.Graphics.Vulkan
 
             _storedBlend = new PipelineColorBlendAttachmentState[Constants.MaxRenderTargets];
 
+            _supportExtDynamic = gd.Capabilities.SupportsExtendedDynamicState;
+
             _newState.Initialize();
         }
 
@@ -685,7 +689,7 @@ namespace Ryujinx.Graphics.Vulkan
         {
             if (texture is TextureView srcTexture)
             {
-                var oldCullMode = _newState.CullMode;
+                var oldCullMode = _supportExtDynamic ? DynamicState.CullMode : _newState.CullMode;
                 var oldStencilTestEnable = _newState.StencilTestEnable;
                 var oldDepthTestEnable = _newState.DepthTestEnable;
                 var oldDepthWriteEnable = _newState.DepthWriteEnable;
@@ -693,7 +697,15 @@ namespace Ryujinx.Graphics.Vulkan
                 var oldViewports = DynamicState.Viewports;
                 var oldViewportsCount = _newState.ViewportsCount;
 
-                _newState.CullMode = CullModeFlags.None;
+                if (_supportExtDynamic)
+                {
+                    DynamicState.SetCullMode(CullModeFlags.None);
+                }
+                else
+                {
+                    _newState.CullMode = CullModeFlags.None;
+                }
+                
                 _newState.StencilTestEnable = false;
                 _newState.DepthTestEnable = false;
                 _newState.DepthWriteEnable = false;
@@ -707,7 +719,15 @@ namespace Ryujinx.Graphics.Vulkan
                     srcRegion,
                     dstRegion);
 
-                _newState.CullMode = oldCullMode;
+                if (_supportExtDynamic)
+                {
+                    DynamicState.SetCullMode(oldCullMode);
+                }
+                else
+                {
+                    _newState.CullMode = oldCullMode;
+                }
+                
                 _newState.StencilTestEnable = oldStencilTestEnable;
                 _newState.DepthTestEnable = oldDepthTestEnable;
                 _newState.DepthWriteEnable = oldDepthWriteEnable;
@@ -878,13 +898,29 @@ namespace Ryujinx.Graphics.Vulkan
 
         public void SetFaceCulling(bool enable, Face face)
         {
-            _newState.CullMode = enable ? face.Convert() : CullModeFlags.None;
+            if (_supportExtDynamic)
+            {
+                DynamicState.SetCullMode(enable ? face.Convert() : CullModeFlags.None);
+            }
+            else
+            {
+                _newState.CullMode = enable ? face.Convert() : CullModeFlags.None;
+            }
+
             SignalStateChange();
         }
 
         public void SetFrontFace(FrontFace frontFace)
         {
-            _newState.FrontFace = frontFace.Convert();
+            if (_supportExtDynamic)
+            {
+                DynamicState.SetFrontFace(frontFace.Convert());
+            }
+            else
+            {
+                _newState.FrontFace = frontFace.Convert();
+            }
+
             SignalStateChange();
         }
 
@@ -1111,23 +1147,39 @@ namespace Ryujinx.Graphics.Vulkan
 
         public void SetStencilTest(StencilTestDescriptor stencilTest)
         {
-            DynamicState.SetStencilMasks(
+            if (Gd.Capabilities.SupportsExtendedDynamicState)
-                (uint)stencilTest.BackFuncMask,
+            {
+                DynamicState.SetStencilOp(
+                    stencilTest.BackSFail.Convert(),
+                    stencilTest.BackDpPass.Convert(),
+                    stencilTest.BackDpFail.Convert(),
+                    stencilTest.BackFunc.Convert(),
+                    stencilTest.FrontSFail.Convert(),
+                    stencilTest.FrontDpPass.Convert(),
+                    stencilTest.FrontDpFail.Convert(),
+                    stencilTest.FrontFunc.Convert());
+            }
+            else
+            {
+                _newState.StencilBackFailOp = stencilTest.BackSFail.Convert();
+                _newState.StencilBackPassOp = stencilTest.BackDpPass.Convert();
+                _newState.StencilBackDepthFailOp = stencilTest.BackDpFail.Convert();
+                _newState.StencilBackCompareOp = stencilTest.BackFunc.Convert();
+                _newState.StencilFrontFailOp = stencilTest.FrontSFail.Convert();
+                _newState.StencilFrontPassOp = stencilTest.FrontDpPass.Convert();
+                _newState.StencilFrontDepthFailOp = stencilTest.FrontDpFail.Convert();
+                _newState.StencilFrontCompareOp = stencilTest.FrontFunc.Convert();
+            }
+            
+            _newState.StencilTestEnable = stencilTest.TestEnable;
+            
+            DynamicState.SetStencilMask((uint)stencilTest.BackFuncMask,
                 (uint)stencilTest.BackMask,
                 (uint)stencilTest.BackFuncRef,
                 (uint)stencilTest.FrontFuncMask,
                 (uint)stencilTest.FrontMask,
                 (uint)stencilTest.FrontFuncRef);
 
-            _newState.StencilTestEnable = stencilTest.TestEnable;
-            _newState.StencilBackFailOp = stencilTest.BackSFail.Convert();
-            _newState.StencilBackPassOp = stencilTest.BackDpPass.Convert();
-            _newState.StencilBackDepthFailOp = stencilTest.BackDpFail.Convert();
-            _newState.StencilBackCompareOp = stencilTest.BackFunc.Convert();
-            _newState.StencilFrontFailOp = stencilTest.FrontSFail.Convert();
-            _newState.StencilFrontPassOp = stencilTest.FrontDpPass.Convert();
-            _newState.StencilFrontDepthFailOp = stencilTest.FrontDpFail.Convert();
-            _newState.StencilFrontCompareOp = stencilTest.FrontFunc.Convert();
             SignalStateChange();
         }
 

src/Ryujinx.Graphics.Vulkan/PipelineConverter.cs

@@ -161,9 +161,6 @@ namespace Ryujinx.Graphics.Vulkan
             pipeline.Initialize();
 
             // It is assumed that Dynamic State is enabled when this conversion is used.
-
-            pipeline.CullMode = state.CullEnable ? state.CullMode.Convert() : CullModeFlags.None;
-
             pipeline.DepthBoundsTestEnable = false; // Not implemented.
 
             pipeline.DepthClampEnable = state.DepthClampEnable;
@@ -173,8 +170,6 @@ namespace Ryujinx.Graphics.Vulkan
             pipeline.DepthCompareOp = state.DepthTest.Func.Convert();
             pipeline.DepthMode = state.DepthMode == DepthMode.MinusOneToOne;
             
-            pipeline.FrontFace = state.FrontFace.Convert();
-
             pipeline.HasDepthStencil = state.DepthStencilEnable;
             pipeline.LineWidth = state.LineWidth;
             pipeline.LogicOpEnable = state.LogicOpEnable;
@@ -203,16 +198,22 @@ namespace Ryujinx.Graphics.Vulkan
             pipeline.DepthBiasEnable = state.BiasEnable != 0;
 
             // Stencil masks and ref are dynamic, so are 0 in the Vulkan pipeline.
+            if (!gd.Capabilities.SupportsExtendedDynamicState)
+            {
+                pipeline.CullMode = state.CullEnable ? state.CullMode.Convert() : CullModeFlags.None;
                 
-            pipeline.StencilFrontFailOp = state.StencilTest.FrontSFail.Convert();
+                pipeline.FrontFace = state.FrontFace.Convert();
-            pipeline.StencilFrontPassOp = state.StencilTest.FrontDpPass.Convert();
-            pipeline.StencilFrontDepthFailOp = state.StencilTest.FrontDpFail.Convert();
-            pipeline.StencilFrontCompareOp = state.StencilTest.FrontFunc.Convert();
                 
-            pipeline.StencilBackFailOp = state.StencilTest.BackSFail.Convert();
+                pipeline.StencilFrontFailOp = state.StencilTest.FrontSFail.Convert();
-            pipeline.StencilBackPassOp = state.StencilTest.BackDpPass.Convert();
+                pipeline.StencilFrontPassOp = state.StencilTest.FrontDpPass.Convert();
-            pipeline.StencilBackDepthFailOp = state.StencilTest.BackDpFail.Convert();
+                pipeline.StencilFrontDepthFailOp = state.StencilTest.FrontDpFail.Convert();
-            pipeline.StencilBackCompareOp = state.StencilTest.BackFunc.Convert();
+                pipeline.StencilFrontCompareOp = state.StencilTest.FrontFunc.Convert();
+
+                pipeline.StencilBackFailOp = state.StencilTest.BackSFail.Convert();
+                pipeline.StencilBackPassOp = state.StencilTest.BackDpPass.Convert();
+                pipeline.StencilBackDepthFailOp = state.StencilTest.BackDpFail.Convert();
+                pipeline.StencilBackCompareOp = state.StencilTest.BackFunc.Convert();
+            }
 
             pipeline.StencilTestEnable = state.StencilTest.TestEnable;
 

src/Ryujinx.Graphics.Vulkan/PipelineDynamicState.cs

@@ -1,5 +1,6 @@
 using Ryujinx.Common.Memory;
 using Silk.NET.Vulkan;
+using System;
 
 namespace Ryujinx.Graphics.Vulkan
 {
@@ -18,12 +19,25 @@ namespace Ryujinx.Graphics.Vulkan
         private uint _frontCompareMask;
         private uint _frontWriteMask;
         private uint _frontReference;
+        private bool _opToo;
+        private StencilOp _backfailop;
+        private StencilOp _backpassop;
+        private StencilOp _backdepthfailop;
+        private CompareOp _backcompareop;
+        private StencilOp _frontfailop;
+        private StencilOp _frontpassop;
+        private StencilOp _frontdepthfailop;
+        private CompareOp _frontcompareop;
 
         private Array4<float> _blendConstants;
 
         public uint ViewportsCount;
         public Array16<Viewport> Viewports;
 
+        public CullModeFlags CullMode;
+        public FrontFace FrontFace;
+
+        [Flags]
         private enum DirtyFlags
         {
             None = 0,
@@ -32,7 +46,9 @@ namespace Ryujinx.Graphics.Vulkan
             Scissor = 1 << 2,
             Stencil = 1 << 3,
             Viewport = 1 << 4,
-            All = Blend | DepthBias | Scissor | Stencil | Viewport,
+            CullMode = 1 << 5,
+            FrontFace = 1 << 6,
+            All = Blend | DepthBias | Scissor | Stencil | Viewport | CullMode | FrontFace,
         }
 
         private DirtyFlags _dirty;
@@ -63,7 +79,7 @@ namespace Ryujinx.Graphics.Vulkan
             _dirty |= DirtyFlags.Scissor;
         }
 
-        public void SetStencilMasks(
+        public void SetStencilMask(
             uint backCompareMask,
             uint backWriteMask,
             uint backReference,
@@ -81,6 +97,27 @@ namespace Ryujinx.Graphics.Vulkan
             _dirty |= DirtyFlags.Stencil;
         }
         
+        public void SetStencilOp(StencilOp backFailOp = default,
+            StencilOp backPassOp = default,
+            StencilOp backDepthFailOp = default,
+            CompareOp backCompareOp = default,
+            StencilOp frontFailOp = default,
+            StencilOp frontPassOp = default,
+            StencilOp frontDepthFailOp = default,
+            CompareOp frontCompareOp = default)
+        {
+            _backfailop = backFailOp;
+            _backpassop = backPassOp;
+            _backdepthfailop = backDepthFailOp;
+            _backcompareop = backCompareOp;
+
+            _frontfailop = frontFailOp;
+            _frontpassop = frontPassOp;
+            _frontdepthfailop = frontDepthFailOp;
+            _frontcompareop = frontCompareOp;
+            _opToo = true;
+        }
+
         public void SetViewport(int index, Viewport viewport)
         {
             Viewports[index] = viewport;
@@ -99,6 +136,20 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
         
+        public void SetCullMode(CullModeFlags cullMode)
+        {
+            CullMode = cullMode;
+            
+            _dirty |= DirtyFlags.CullMode;
+        }
+        
+        public void SetFrontFace(FrontFace frontFace)
+        {
+            FrontFace = frontFace;
+            
+            _dirty |= DirtyFlags.FrontFace;
+        }
+
         public void ForceAllDirty()
         {
             _dirty = DirtyFlags.All;
@@ -123,7 +174,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             if (_dirty.HasFlag(DirtyFlags.Stencil))
             {
-                RecordStencilMasks(api, commandBuffer);
+                RecordStencil(api, commandBuffer);
             }
 
             if (_dirty.HasFlag(DirtyFlags.Viewport))
@@ -131,6 +182,16 @@ namespace Ryujinx.Graphics.Vulkan
                 RecordViewport(api, commandBuffer);
             }
             
+            if (_dirty.HasFlag(DirtyFlags.CullMode))
+            {
+                RecordCullMode(api, commandBuffer);
+            }
+            
+            if (_dirty.HasFlag(DirtyFlags.FrontFace))
+            {
+                RecordFrontFace(api, commandBuffer);
+            }
+
             _dirty = DirtyFlags.None;
         }
 
@@ -152,8 +213,16 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
         
-        private readonly void RecordStencilMasks(Vk api, CommandBuffer commandBuffer)
+        private readonly void RecordStencil(Vk api, CommandBuffer commandBuffer)
         {
+            if (_opToo)
+            {
+                api.CmdSetStencilOp(commandBuffer, StencilFaceFlags.FaceBackBit, _backfailop, _backpassop,
+                    _backdepthfailop, _backcompareop);
+                api.CmdSetStencilOp(commandBuffer, StencilFaceFlags.FaceFrontBit, _frontfailop, _frontpassop,
+                    _frontdepthfailop, _frontcompareop);
+            }
+            
             api.CmdSetStencilCompareMask(commandBuffer, StencilFaceFlags.FaceBackBit, _backCompareMask);
             api.CmdSetStencilWriteMask(commandBuffer, StencilFaceFlags.FaceBackBit, _backWriteMask);
             api.CmdSetStencilReference(commandBuffer, StencilFaceFlags.FaceBackBit, _backReference);
@@ -169,5 +238,15 @@ namespace Ryujinx.Graphics.Vulkan
                 api.CmdSetViewport(commandBuffer, 0, ViewportsCount, Viewports.AsSpan());
             }
         }
+        
+        private void RecordCullMode(Vk api, CommandBuffer commandBuffer)
+        {
+            api.CmdSetCullMode(commandBuffer, CullMode);
+        }
+        
+        private void RecordFrontFace(Vk api, CommandBuffer commandBuffer)
+        {
+            api.CmdSetFrontFace(commandBuffer, FrontFace);
+        }
     }
 }

src/Ryujinx.Graphics.Vulkan/PipelineState.cs

@@ -405,6 +405,8 @@ namespace Ryujinx.Graphics.Vulkan
                 UpdateVertexAttributeDescriptions(gd);
             }
             
+            bool supportsExtDynamicState = gd.Capabilities.SupportsExtendedDynamicState;
+
             fixed (VertexInputAttributeDescription* pVertexAttributeDescriptions = &Internal.VertexAttributeDescriptions[0])
             fixed (VertexInputAttributeDescription* pVertexAttributeDescriptions2 = &_vertexAttributeDescriptions2[0])
             fixed (VertexInputBindingDescription* pVertexBindingDescriptions = &Internal.VertexBindingDescriptions[0])
@@ -467,11 +469,15 @@ namespace Ryujinx.Graphics.Vulkan
                     RasterizerDiscardEnable = RasterizerDiscardEnable,
                     PolygonMode = PolygonMode,
                     LineWidth = LineWidth,
-                    CullMode = CullMode,
-                    FrontFace = FrontFace,
                     DepthBiasEnable = DepthBiasEnable,
                 };
 
+                if (!supportsExtDynamicState)
+                {
+                    rasterizationState.CullMode = CullMode;
+                    rasterizationState.FrontFace = FrontFace;
+                }
+
                 var viewportState = new PipelineViewportStateCreateInfo
                 {
                     SType = StructureType.PipelineViewportStateCreateInfo,
@@ -500,24 +506,6 @@ namespace Ryujinx.Graphics.Vulkan
                     AlphaToOneEnable = AlphaToOneEnable,
                 };
 
-                var stencilFront = new StencilOpState(
-                    StencilFrontFailOp,
-                    StencilFrontPassOp,
-                    StencilFrontDepthFailOp,
-                    StencilFrontCompareOp,
-                    null,
-                    null,
-                    null);
-
-                var stencilBack = new StencilOpState(
-                    StencilBackFailOp,
-                    StencilBackPassOp,
-                    StencilBackDepthFailOp,
-                    StencilBackCompareOp,
-                    null,
-                    null,
-                    null);
-
                 var depthStencilState = new PipelineDepthStencilStateCreateInfo
                 {
                     SType = StructureType.PipelineDepthStencilStateCreateInfo,
@@ -526,15 +514,37 @@ namespace Ryujinx.Graphics.Vulkan
                     DepthCompareOp = DepthCompareOp,
                     DepthBoundsTestEnable = DepthBoundsTestEnable,
                     StencilTestEnable = StencilTestEnable,
-                    Front = stencilFront,
-                    Back = stencilBack,
                     MinDepthBounds = MinDepthBounds,
                     MaxDepthBounds = MaxDepthBounds,
                 };
                 
+                if (!supportsExtDynamicState)
+                {
+                    var stencilFront = new StencilOpState(
+                        StencilFrontFailOp,
+                        StencilFrontPassOp,
+                        StencilFrontDepthFailOp,
+                        StencilFrontCompareOp,
+                        null,
+                        null,
+                        null);
+
+                    var stencilBack = new StencilOpState(
+                        StencilBackFailOp,
+                        StencilBackPassOp,
+                        StencilBackDepthFailOp,
+                        StencilBackCompareOp,
+                        null,
+                        null,
+                        null);
+
+                    depthStencilState.Front = stencilFront;
+                    depthStencilState.Back = stencilBack;
+                }
+
                 uint blendEnables = 0;
 
-                if (gd.IsMoltenVk && Internal.AttachmentIntegerFormatMask != 0)
+                if (isMoltenVk && Internal.AttachmentIntegerFormatMask != 0)
                 {
                     // Blend can't be enabled for integer formats, so let's make sure it is disabled.
                     uint attachmentIntegerFormatMask = Internal.AttachmentIntegerFormatMask;
@@ -579,8 +589,7 @@ namespace Ryujinx.Graphics.Vulkan
                     colorBlendState.PNext = &colorBlendAdvancedState;
                 }
                 
-                bool supportsExtDynamicState = gd.Capabilities.SupportsExtendedDynamicState;
+                int dynamicStatesCount = supportsExtDynamicState ? (isMoltenVk ? 10 : 11) : 7;
-                int dynamicStatesCount = supportsExtDynamicState ? 8 : 7;
 
                 DynamicState* dynamicStates = stackalloc DynamicState[dynamicStatesCount];
 
@@ -594,7 +603,13 @@ namespace Ryujinx.Graphics.Vulkan
 
                 if (supportsExtDynamicState)
                 {
-                    dynamicStates[7] = DynamicState.VertexInputBindingStrideExt;
+                    int index = 7;
+                    if (!isMoltenVk) {
+                        dynamicStates[index++] = DynamicState.VertexInputBindingStrideExt;
+                    }
+                    dynamicStates[index++] = DynamicState.CullMode;
+                    dynamicStates[index++] = DynamicState.FrontFace;
+                    dynamicStates[index] = DynamicState.StencilOp;
                 }
 
                 var pipelineDynamicStateCreateInfo = new PipelineDynamicStateCreateInfo