Optimize texture format conversion, and MethodCopyBuffer (#1274)

* Improve performance when converting texture formats.

Still more work to do.

* Speed up buffer -> texture copies.

No longer copies byte by byte. Fast path when formats are identical.

* Fix a few things, 64 byte block fast copy.

* Spacing cleanup, unrelated change.

* Fix base offset calculation for region copies.

* Fix Linear -> BlockLinear

* Fix some nits. (part 1 of review feedback)

* Use a generic version of the Convert* functions rather than lambdas.

This is some real monkey's paw shit.

* Remove unnecessary span constructor.

* Revert "Use a generic version of the Convert* functions rather than lambdas."

This reverts commit aa43dcfbe8.

* Fix bug with rectangle destination writing, better rectangle calculation for linear textures.

Ryujinx.Graphics.Gpu/Engine/MethodCopyBuffer.cs

@@ -1,6 +1,7 @@
 using Ryujinx.Graphics.Gpu.State;
 using Ryujinx.Graphics.Texture;
 using System;
+using System.Runtime.Intrinsics;
 
 namespace Ryujinx.Graphics.Gpu.Engine
 {
@@ -56,20 +57,59 @@ namespace Ryujinx.Graphics.Gpu.Engine
                 ulong srcBaseAddress = _context.MemoryManager.Translate(cbp.SrcAddress.Pack());
                 ulong dstBaseAddress = _context.MemoryManager.Translate(cbp.DstAddress.Pack());
 
+                (int srcBaseOffset, int srcSize) = srcCalculator.GetRectangleRange(src.RegionX, src.RegionY, cbp.XCount, cbp.YCount);
+                (int dstBaseOffset, int dstSize) = dstCalculator.GetRectangleRange(dst.RegionX, dst.RegionY, cbp.XCount, cbp.YCount);
+
+                ReadOnlySpan<byte> srcSpan = _context.PhysicalMemory.GetSpan(srcBaseAddress + (ulong)srcBaseOffset, srcSize);
+                Span<byte> dstSpan = _context.PhysicalMemory.GetSpan(dstBaseAddress + (ulong)dstBaseOffset, dstSize).ToArray();
+
+                bool completeSource = src.RegionX == 0 && src.RegionY == 0 && src.Width == cbp.XCount && src.Height == cbp.YCount;
+                bool completeDest = dst.RegionX == 0 && dst.RegionY == 0 && dst.Width == cbp.XCount && dst.Height == cbp.YCount;
+
+                if (completeSource && completeDest && srcCalculator.LayoutMatches(dstCalculator))
+                {
+                    srcSpan.CopyTo(dstSpan); // No layout conversion has to be performed, just copy the data entirely.
+                }
+                else 
+                {
+                    unsafe bool Convert<T>(Span<byte> dstSpan, ReadOnlySpan<byte> srcSpan) where T : unmanaged
+                    {
+                        fixed (byte* dstPtr = dstSpan, srcPtr = srcSpan)
+                        {
+                            byte* dstBase = dstPtr - dstBaseOffset; // Layout offset is relative to the base, so we need to subtract the span's offset.
+                            byte* srcBase = srcPtr - srcBaseOffset;
+
                             for (int y = 0; y < cbp.YCount; y++)
+                            {
+                                srcCalculator.SetY(src.RegionY + y);
+                                dstCalculator.SetY(dst.RegionY + y);
+
                                 for (int x = 0; x < cbp.XCount; x++)
                                 {
-                    int srcOffset = srcCalculator.GetOffset(src.RegionX + x, src.RegionY + y);
+                                    int srcOffset = srcCalculator.GetOffset(src.RegionX + x);
-                    int dstOffset = dstCalculator.GetOffset(dst.RegionX + x, dst.RegionY + y);
+                                    int dstOffset = dstCalculator.GetOffset(dst.RegionX + x);
 
-                    ulong srcAddress = srcBaseAddress + (ulong)srcOffset;
+                                    *(T*)(dstBase + dstOffset) = *(T*)(srcBase + srcOffset);
-                    ulong dstAddress = dstBaseAddress + (ulong)dstOffset;
-
-                    ReadOnlySpan<byte> pixel = _context.PhysicalMemory.GetSpan(srcAddress, srcBpp);
-
-                    _context.PhysicalMemory.Write(dstAddress, pixel);
                                 }
                             }
+                        }
+                        return true;
+                    }
+
+                    bool _ = srcBpp switch
+                    {
+                        1 => Convert<byte>(dstSpan, srcSpan),
+                        2 => Convert<ushort>(dstSpan, srcSpan),
+                        4 => Convert<uint>(dstSpan, srcSpan),
+                        8 => Convert<ulong>(dstSpan, srcSpan),
+                        12 => Convert<Bpp12Pixel>(dstSpan, srcSpan),
+                        16 => Convert<Vector128<byte>>(dstSpan, srcSpan),
+                        _ => throw new NotSupportedException($"Unable to copy ${srcBpp} bpp pixel format.")
+                    };
+                }
+
+                _context.PhysicalMemory.Write(dstBaseAddress + (ulong)dstBaseOffset, dstSpan);
+            }
             else
             {
                 // Buffer to buffer copy.

Ryujinx.Graphics.Texture/BlockLinearLayout.cs

@@ -33,6 +33,11 @@ namespace Ryujinx.Graphics.Texture
         private int _robSize;
         private int _sliceSize;
 
+        // Variables for built in iteration.
+        private int _yPart;
+        private int _yzPart;
+        private int _zPart;
+
         public BlockLinearLayout(
             int width,
             int height,
@@ -97,5 +102,94 @@ namespace Ryujinx.Graphics.Texture
 
             return offset;
         }
+
+        public (int offset, int size) GetRectangleRange(int x, int y, int width, int height)
+        {
+            // Justification:
+            // The 2D offset is a combination of separate x and y parts.
+            // Both components increase with input and never overlap bits.
+            // Therefore for each component, the minimum input value is the lowest that component can go.
+            // Minimum total value is minimum X component + minimum Y component. Similar goes for maximum.
+
+            int start = GetOffset(x, y, 0);
+            int end = GetOffset(x + width - 1, y + height - 1, 0) + _texBpp; // Cover the last pixel.
+            return (start, end - start);
+        }
+
+        public bool LayoutMatches(BlockLinearLayout other)
+        {
+            return _robSize == other._robSize &&
+                   _sliceSize == other._sliceSize &&
+                   _texBpp == other._texBpp &&
+                   _bhMask == other._bhMask &&
+                   _bdMask == other._bdMask;
+        }
+
+        // Functions for built in iteration.
+        // Components of the offset can be updated separately, and combined to save some time.
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void SetY(int y)
+        {
+            int yh = y / GobHeight;
+            int offset = (yh >> _bhShift) * _robSize;
+
+            offset += (yh & _bhMask) * GobSize;
+
+            offset += ((y & 0x07) >> 1) << 6;
+            offset += ((y & 0x01) >> 0) << 4;
+
+            _yPart = offset;
+            _yzPart = offset + _zPart;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void SetZ(int z)
+        {
+            int offset = (z >> _bdShift) * _sliceSize;
+
+            offset += ((z & _bdMask) * GobSize) << _bhShift;
+
+            _zPart = offset;
+            _yzPart = offset + _yPart;
+        }
+
+        /// <summary>
+        /// Optimized conversion for line offset in bytes to an absolute offset. Input x must be divisible by 16.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetOffsetWithLineOffset16(int x)
+        {
+            int offset = (x / GobStride) << _xShift;
+
+            offset += ((x & 0x3f) >> 5) << 8;
+            offset += ((x & 0x1f) >> 4) << 5;
+
+            return offset + _yzPart;
+        }
+
+        /// <summary>
+        /// Optimized conversion for line offset in bytes to an absolute offset. Input x must be divisible by 64.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetOffsetWithLineOffset64(int x)
+        {
+            int offset = (x / GobStride) << _xShift;
+
+            return offset + _yzPart;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetOffset(int x)
+        {
+            x <<= _bppShift;
+            int offset = (x / GobStride) << _xShift;
+
+            offset += ((x & 0x3f) >> 5) << 8;
+            offset += ((x & 0x1f) >> 4) << 5;
+            offset += (x & 0x0f);
+
+            return offset + _yzPart;
+        }
     }
 }

Ryujinx.Graphics.Texture/Bpp12Pixel.cs

@@ -0,0 +1,11 @@
+using System.Runtime.InteropServices;
+
+namespace Ryujinx.Graphics.Texture
+{
+    [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 12)]
+    public struct Bpp12Pixel
+    {
+        private ulong _elem1;
+        private uint _elem2;
+    }
+}

Ryujinx.Graphics.Texture/LayoutConverter.cs

@@ -1,6 +1,6 @@
 using Ryujinx.Common;
 using System;
-
+using System.Runtime.Intrinsics;
 using static Ryujinx.Graphics.Texture.BlockLinearConstants;
 
 namespace Ryujinx.Graphics.Texture
@@ -64,11 +64,14 @@ namespace Ryujinx.Graphics.Texture
                 }
 
                 int strideTrunc = BitUtils.AlignDown(w * bytesPerPixel, 16);
+                int strideTrunc64 = BitUtils.AlignDown(w * bytesPerPixel, 64);
 
                 int xStart = strideTrunc / bytesPerPixel;
 
                 int stride = BitUtils.AlignUp(w * bytesPerPixel, HostStrideAlignment);
 
+                int outStrideGap = stride - w * bytesPerPixel;
+
                 int alignment = gobWidth;
 
                 if (d < gobBlocksInZ || w <= gobWidth || h <= gobHeight)
@@ -86,36 +89,74 @@ namespace Ryujinx.Graphics.Texture
                     mipGobBlocksInZ,
                     bytesPerPixel);
 
+                unsafe bool Convert<T>(Span<byte> output, ReadOnlySpan<byte> data) where T : unmanaged
+                {
+                    fixed (byte* outputPtr = output, dataPtr = data)
+                    {
+                        byte* outPtr = outputPtr + outOffs;
                         for (int layer = 0; layer < layers; layer++)
                         {
-                    int inBaseOffset = layer * sizeInfo.LayerSize + sizeInfo.GetMipOffset(level);
+                            byte* inBaseOffset = dataPtr + (layer * sizeInfo.LayerSize + sizeInfo.GetMipOffset(level));
 
                             for (int z = 0; z < d; z++)
+                            {
+                                layoutConverter.SetZ(z);
                                 for (int y = 0; y < h; y++)
                                 {
-                        for (int x = 0; x < strideTrunc; x += 16)
+                                    layoutConverter.SetY(y);
+
+                                    for (int x = 0; x < strideTrunc64; x += 64, outPtr += 64)
                                     {
-                            int offset = inBaseOffset + layoutConverter.GetOffsetWithLineOffset(x, y, z);
+                                        byte* offset = inBaseOffset + layoutConverter.GetOffsetWithLineOffset64(x);
+                                        byte* offset2 = offset + 0x20;
+                                        byte* offset3 = offset + 0x100;
+                                        byte* offset4 = offset + 0x120;
 
-                            Span<byte> dest = output.Slice(outOffs + x, 16);
+                                        Vector128<byte> value = *(Vector128<byte>*)offset;
+                                        Vector128<byte> value2 = *(Vector128<byte>*)offset2;
+                                        Vector128<byte> value3 = *(Vector128<byte>*)offset3;
+                                        Vector128<byte> value4 = *(Vector128<byte>*)offset4;
 
-                            data.Slice(offset, 16).CopyTo(dest);
+                                        *(Vector128<byte>*)outPtr = value;
+                                        *(Vector128<byte>*)(outPtr + 16) = value2;
+                                        *(Vector128<byte>*)(outPtr + 32) = value3;
+                                        *(Vector128<byte>*)(outPtr + 48) = value4;
                                     }
 
-                        for (int x = xStart; x < w; x++)
+                                    for (int x = strideTrunc64; x < strideTrunc; x += 16, outPtr += 16)
                                     {
-                            int offset = inBaseOffset + layoutConverter.GetOffset(x, y, z);
+                                        byte* offset = inBaseOffset + layoutConverter.GetOffsetWithLineOffset16(x);
 
-                            Span<byte> dest = output.Slice(outOffs + x * bytesPerPixel, bytesPerPixel);
+                                        *(Vector128<byte>*)outPtr = *(Vector128<byte>*)offset;
-
-                            data.Slice(offset, bytesPerPixel).CopyTo(dest);
                                     }
 
-                        outOffs += stride;
+                                    for (int x = xStart; x < w; x++, outPtr += bytesPerPixel)
+                                    {
+                                        byte* offset = inBaseOffset + layoutConverter.GetOffset(x);
+
+                                        *(T*)outPtr = *(T*)offset;
+                                    }
+
+                                    outPtr += outStrideGap;
                                 }
                             }
                         }
+                        outOffs += stride * h * d * layers;
+                    }
+                    return true;
+                }
 
+                bool _ = bytesPerPixel switch
+                {
+                    1 => Convert<byte>(output, data),
+                    2 => Convert<ushort>(output, data),
+                    4 => Convert<uint>(output, data),
+                    8 => Convert<ulong>(output, data),
+                    12 => Convert<Bpp12Pixel>(output, data),
+                    16 => Convert<Vector128<byte>>(output, data),
+                    _ => throw new NotSupportedException($"Unable to convert ${bytesPerPixel} bpp pixel format.")
+                };
+            }
             return output;
         }
 
@@ -132,22 +173,18 @@ namespace Ryujinx.Graphics.Texture
             int h = BitUtils.DivRoundUp(height, blockHeight);
 
             int outStride = BitUtils.AlignUp(w * bytesPerPixel, HostStrideAlignment);
+            int lineSize = w * bytesPerPixel;
 
             Span<byte> output = new byte[h * outStride];
 
             int outOffs = 0;
+            int inOffs = 0;
 
             for (int y = 0; y < h; y++)
             {
-                for (int x = 0; x < w; x++)
+                data.Slice(inOffs, lineSize).CopyTo(output.Slice(outOffs, lineSize));
-                {
-                    int offset = y * stride + x * bytesPerPixel;
-
-                    Span<byte> dest = output.Slice(outOffs + x * bytesPerPixel, bytesPerPixel);
-
-                    data.Slice(offset, bytesPerPixel).CopyTo(dest);
-                }
 
+                inOffs += stride;
                 outOffs += outStride;
             }
 
@@ -198,8 +235,15 @@ namespace Ryujinx.Graphics.Texture
                     mipGobBlocksInZ >>= 1;
                 }
 
+                int strideTrunc = BitUtils.AlignDown(w * bytesPerPixel, 16);
+                int strideTrunc64 = BitUtils.AlignDown(w * bytesPerPixel, 64);
+
+                int xStart = strideTrunc / bytesPerPixel;
+
                 int stride = BitUtils.AlignUp(w * bytesPerPixel, HostStrideAlignment);
 
+                int inStrideGap = stride - w * bytesPerPixel;
+
                 int alignment = gobWidth;
 
                 if (d < gobBlocksInZ || w <= gobWidth || h <= gobHeight)
@@ -217,26 +261,74 @@ namespace Ryujinx.Graphics.Texture
                     mipGobBlocksInZ,
                     bytesPerPixel);
 
+                unsafe bool Convert<T>(Span<byte> output, ReadOnlySpan<byte> data) where T : unmanaged
+                {
+                    fixed (byte* outputPtr = output, dataPtr = data)
+                    {
+                        byte* inPtr = dataPtr + inOffs;
                         for (int layer = 0; layer < layers; layer++)
                         {
-                    int outBaseOffset = layer * sizeInfo.LayerSize + sizeInfo.GetMipOffset(level);
+                            byte* outBaseOffset = outputPtr + (layer * sizeInfo.LayerSize + sizeInfo.GetMipOffset(level));
 
                             for (int z = 0; z < d; z++)
+                            {
+                                layoutConverter.SetZ(z);
                                 for (int y = 0; y < h; y++)
                                 {
-                        for (int x = 0; x < w; x++)
+                                    layoutConverter.SetY(y);
+
+                                    for (int x = 0; x < strideTrunc64; x += 64, inPtr += 64)
                                     {
-                            int offset = outBaseOffset + layoutConverter.GetOffset(x, y, z);
+                                        byte* offset = outBaseOffset + layoutConverter.GetOffsetWithLineOffset64(x);
+                                        byte* offset2 = offset + 0x20;
+                                        byte* offset3 = offset + 0x100;
+                                        byte* offset4 = offset + 0x120;
 
-                            Span<byte> dest = output.Slice(offset, bytesPerPixel);
+                                        Vector128<byte> value = *(Vector128<byte>*)inPtr;
+                                        Vector128<byte> value2 = *(Vector128<byte>*)(inPtr + 16);
+                                        Vector128<byte> value3 = *(Vector128<byte>*)(inPtr + 32);
+                                        Vector128<byte> value4 = *(Vector128<byte>*)(inPtr + 48);
 
-                            data.Slice(inOffs + x * bytesPerPixel, bytesPerPixel).CopyTo(dest);
+                                        *(Vector128<byte>*)offset = value;
+                                        *(Vector128<byte>*)offset2 = value2;
+                                        *(Vector128<byte>*)offset3 = value3;
+                                        *(Vector128<byte>*)offset4 = value4;
                                     }
 
-                        inOffs += stride;
+                                    for (int x = strideTrunc64; x < strideTrunc; x += 16, inPtr += 16)
+                                    {
+                                        byte* offset = outBaseOffset + layoutConverter.GetOffsetWithLineOffset16(x);
+
+                                        *(Vector128<byte>*)offset = *(Vector128<byte>*)inPtr;
+                                    }
+
+                                    for (int x = xStart; x < w; x++, inPtr += bytesPerPixel)
+                                    {
+                                        byte* offset = outBaseOffset + layoutConverter.GetOffset(x);
+
+                                        *(T*)offset = *(T*)inPtr;
+                                    }
+
+                                    inPtr += inStrideGap;
                                 }
                             }
                         }
+                        inOffs += stride * h * d * layers;
+                    }
+                    return true;
+                }
+
+                bool _ = bytesPerPixel switch
+                {
+                    1 => Convert<byte>(output, data),
+                    2 => Convert<ushort>(output, data),
+                    4 => Convert<uint>(output, data),
+                    8 => Convert<ulong>(output, data),
+                    12 => Convert<Bpp12Pixel>(output, data),
+                    16 => Convert<Vector128<byte>>(output, data),
+                    _ => throw new NotSupportedException($"Unable to convert ${bytesPerPixel} bpp pixel format.")
+                };
+            }
 
             return output;
         }
@@ -254,23 +346,19 @@ namespace Ryujinx.Graphics.Texture
             int h = BitUtils.DivRoundUp(height, blockHeight);
 
             int inStride = BitUtils.AlignUp(w * bytesPerPixel, HostStrideAlignment);
+            int lineSize = width * bytesPerPixel;
 
             Span<byte> output = new byte[h * stride];
 
             int inOffs = 0;
+            int outOffs = 0;
 
             for (int y = 0; y < h; y++)
             {
-                for (int x = 0; x < w; x++)
+                data.Slice(inOffs, lineSize).CopyTo(output.Slice(outOffs, lineSize));
-                {
-                    int offset = y * stride + x * bytesPerPixel;
-
-                    Span<byte> dest = output.Slice(offset, bytesPerPixel);
-
-                    data.Slice(inOffs + x * bytesPerPixel, bytesPerPixel).CopyTo(dest);
-                }
 
                 inOffs += inStride;
+                outOffs += stride;
             }
 
             return output;

Ryujinx.Graphics.Texture/OffsetCalculator.cs

@@ -1,17 +1,22 @@
 using Ryujinx.Common;
-
+using System.Runtime.CompilerServices;
 using static Ryujinx.Graphics.Texture.BlockLinearConstants;
 
 namespace Ryujinx.Graphics.Texture
 {
     public class OffsetCalculator
     {
+        private int  _width;
+        private int  _height;
         private int  _stride;
         private bool _isLinear;
         private int  _bytesPerPixel;
 
         private BlockLinearLayout _layoutConverter;
 
+        // Variables for built in iteration.
+        private int _yPart;
+
         public OffsetCalculator(
             int  width,
             int  height,
@@ -20,6 +25,8 @@ namespace Ryujinx.Graphics.Texture
             int  gobBlocksInY,
             int  bytesPerPixel)
         {
+            _width         = width;
+            _height        = height;
             _stride        = stride;
             _isLinear      = isLinear;
             _bytesPerPixel = bytesPerPixel;
@@ -40,6 +47,18 @@ namespace Ryujinx.Graphics.Texture
             }
         }
 
+        public void SetY(int y)
+        {
+            if (_isLinear)
+            {
+                _yPart = y * _stride;
+            }
+            else
+            {
+                _layoutConverter.SetY(y);
+            }
+        }
+
         public int GetOffset(int x, int y)
         {
             if (_isLinear)
@@ -51,5 +70,48 @@ namespace Ryujinx.Graphics.Texture
                 return _layoutConverter.GetOffset(x, y, 0);
             }
         }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetOffset(int x)
+        {
+            if (_isLinear)
+            {
+                return x * _bytesPerPixel + _yPart;
+            }
+            else
+            {
+                return _layoutConverter.GetOffset(x);
+            }
+        }
+
+        public (int offset, int size) GetRectangleRange(int x, int y, int width, int height)
+        {
+            if (_isLinear)
+            {
+                int start = y * _stride + x * _bytesPerPixel;
+                int end = (y + height - 1) * _stride + (x + width) * _bytesPerPixel;
+                return (start, end - start);
+            }
+            else
+            {
+                return _layoutConverter.GetRectangleRange(x, y, width, height);
+            }
+        }
+
+        public bool LayoutMatches(OffsetCalculator other)
+        {
+            if (_isLinear)
+            {
+                return other._isLinear &&
+                       _width == other._width &&
+                       _height == other._height &&
+                       _stride == other._stride &&
+                       _bytesPerPixel == other._bytesPerPixel;
+            }
+            else
+            {
+                return !other._isLinear && _layoutConverter.LayoutMatches(other._layoutConverter);
+            }
+        }
     }
 }