Truncate vertex attribute format if it exceeds stride on MoltenVK (#5094)

* Truncate vertex attribute format if it exceeds stride on MoltenVK

* Fix BGR format

* Move vertex attribute check to pipeline creation to avoid costs

* No need for this to be public

.github/workflows/release.yml

@@ -33,7 +33,7 @@ jobs:
         shell: bash
 
       - name: Create tag
-        uses: actions/github-script@v5
+        uses: actions/github-script@v6
         with:
           script: |
             github.rest.git.createRef({

src/Ryujinx.Graphics.Vulkan/FormatTable.cs

@@ -168,5 +168,223 @@ namespace Ryujinx.Graphics.Vulkan
         {
             return _table[(int)format];
         }
+
+        public static int GetAttributeFormatSize(VkFormat format)
+        {
+            switch (format)
+            {
+                case VkFormat.R8Unorm:
+                case VkFormat.R8SNorm:
+                case VkFormat.R8Uint:
+                case VkFormat.R8Sint:
+                case VkFormat.R8Uscaled:
+                case VkFormat.R8Sscaled:
+                    return 1;
+
+                case VkFormat.R8G8Unorm:
+                case VkFormat.R8G8SNorm:
+                case VkFormat.R8G8Uint:
+                case VkFormat.R8G8Sint:
+                case VkFormat.R8G8Uscaled:
+                case VkFormat.R8G8Sscaled:
+                case VkFormat.R16Sfloat:
+                case VkFormat.R16Unorm:
+                case VkFormat.R16SNorm:
+                case VkFormat.R16Uint:
+                case VkFormat.R16Sint:
+                case VkFormat.R16Uscaled:
+                case VkFormat.R16Sscaled:
+                    return 2;
+
+                case VkFormat.R8G8B8Unorm:
+                case VkFormat.R8G8B8SNorm:
+                case VkFormat.R8G8B8Uint:
+                case VkFormat.R8G8B8Sint:
+                case VkFormat.R8G8B8Uscaled:
+                case VkFormat.R8G8B8Sscaled:
+                    return 3;
+
+                case VkFormat.R8G8B8A8Unorm:
+                case VkFormat.R8G8B8A8SNorm:
+                case VkFormat.R8G8B8A8Uint:
+                case VkFormat.R8G8B8A8Sint:
+                case VkFormat.R8G8B8A8Srgb:
+                case VkFormat.R8G8B8A8Uscaled:
+                case VkFormat.R8G8B8A8Sscaled:
+                case VkFormat.B8G8R8A8Unorm:
+                case VkFormat.B8G8R8A8Srgb:
+                case VkFormat.R16G16Sfloat:
+                case VkFormat.R16G16Unorm:
+                case VkFormat.R16G16SNorm:
+                case VkFormat.R16G16Uint:
+                case VkFormat.R16G16Sint:
+                case VkFormat.R16G16Uscaled:
+                case VkFormat.R16G16Sscaled:
+                case VkFormat.R32Sfloat:
+                case VkFormat.R32Uint:
+                case VkFormat.R32Sint:
+                case VkFormat.A2B10G10R10UnormPack32:
+                case VkFormat.A2B10G10R10UintPack32:
+                case VkFormat.B10G11R11UfloatPack32:
+                case VkFormat.E5B9G9R9UfloatPack32:
+                case VkFormat.A2B10G10R10SNormPack32:
+                case VkFormat.A2B10G10R10SintPack32:
+                case VkFormat.A2B10G10R10UscaledPack32:
+                case VkFormat.A2B10G10R10SscaledPack32:
+                    return 4;
+
+                case VkFormat.R16G16B16Sfloat:
+                case VkFormat.R16G16B16Unorm:
+                case VkFormat.R16G16B16SNorm:
+                case VkFormat.R16G16B16Uint:
+                case VkFormat.R16G16B16Sint:
+                case VkFormat.R16G16B16Uscaled:
+                case VkFormat.R16G16B16Sscaled:
+                    return 6;
+
+                case VkFormat.R16G16B16A16Sfloat:
+                case VkFormat.R16G16B16A16Unorm:
+                case VkFormat.R16G16B16A16SNorm:
+                case VkFormat.R16G16B16A16Uint:
+                case VkFormat.R16G16B16A16Sint:
+                case VkFormat.R16G16B16A16Uscaled:
+                case VkFormat.R16G16B16A16Sscaled:
+                case VkFormat.R32G32Sfloat:
+                case VkFormat.R32G32Uint:
+                case VkFormat.R32G32Sint:
+                    return 8;
+
+                case VkFormat.R32G32B32Sfloat:
+                case VkFormat.R32G32B32Uint:
+                case VkFormat.R32G32B32Sint:
+                    return 12;
+
+                case VkFormat.R32G32B32A32Sfloat:
+                case VkFormat.R32G32B32A32Uint:
+                case VkFormat.R32G32B32A32Sint:
+                    return 16;
+            }
+
+            return 1;
+        }
+
+        public static VkFormat DropLastComponent(VkFormat format)
+        {
+            switch (format)
+            {
+                case VkFormat.R8G8Unorm:
+                    return VkFormat.R8Unorm;
+                case VkFormat.R8G8SNorm:
+                    return VkFormat.R8SNorm;
+                case VkFormat.R8G8Uint:
+                    return VkFormat.R8Uint;
+                case VkFormat.R8G8Sint:
+                    return VkFormat.R8Sint;
+                case VkFormat.R8G8Uscaled:
+                    return VkFormat.R8Uscaled;
+                case VkFormat.R8G8Sscaled:
+                    return VkFormat.R8Sscaled;
+
+                case VkFormat.R8G8B8Unorm:
+                    return VkFormat.R8G8Unorm;
+                case VkFormat.R8G8B8SNorm:
+                    return VkFormat.R8G8SNorm;
+                case VkFormat.R8G8B8Uint:
+                    return VkFormat.R8G8Uint;
+                case VkFormat.R8G8B8Sint:
+                    return VkFormat.R8G8Sint;
+                case VkFormat.R8G8B8Uscaled:
+                    return VkFormat.R8G8Uscaled;
+                case VkFormat.R8G8B8Sscaled:
+                    return VkFormat.R8G8Sscaled;
+
+                case VkFormat.R8G8B8A8Unorm:
+                    return VkFormat.R8G8B8Unorm;
+                case VkFormat.R8G8B8A8SNorm:
+                    return VkFormat.R8G8B8SNorm;
+                case VkFormat.R8G8B8A8Uint:
+                    return VkFormat.R8G8B8Uint;
+                case VkFormat.R8G8B8A8Sint:
+                    return VkFormat.R8G8B8Sint;
+                case VkFormat.R8G8B8A8Srgb:
+                    return VkFormat.R8G8B8Srgb;
+                case VkFormat.R8G8B8A8Uscaled:
+                    return VkFormat.R8G8B8Uscaled;
+                case VkFormat.R8G8B8A8Sscaled:
+                    return VkFormat.R8G8B8Sscaled;
+                case VkFormat.B8G8R8A8Unorm:
+                    return VkFormat.B8G8R8Unorm;
+                case VkFormat.B8G8R8A8Srgb:
+                    return VkFormat.B8G8R8Srgb;
+
+                case VkFormat.R16G16Sfloat:
+                    return VkFormat.R16Sfloat;
+                case VkFormat.R16G16Unorm:
+                    return VkFormat.R16Unorm;
+                case VkFormat.R16G16SNorm:
+                    return VkFormat.R16SNorm;
+                case VkFormat.R16G16Uint:
+                    return VkFormat.R16Uint;
+                case VkFormat.R16G16Sint:
+                    return VkFormat.R16Sint;
+                case VkFormat.R16G16Uscaled:
+                    return VkFormat.R16Uscaled;
+                case VkFormat.R16G16Sscaled:
+                    return VkFormat.R16Sscaled;
+
+                case VkFormat.R16G16B16Sfloat:
+                    return VkFormat.R16G16Sfloat;
+                case VkFormat.R16G16B16Unorm:
+                    return VkFormat.R16G16Unorm;
+                case VkFormat.R16G16B16SNorm:
+                    return VkFormat.R16G16SNorm;
+                case VkFormat.R16G16B16Uint:
+                    return VkFormat.R16G16Uint;
+                case VkFormat.R16G16B16Sint:
+                    return VkFormat.R16G16Sint;
+                case VkFormat.R16G16B16Uscaled:
+                    return VkFormat.R16G16Uscaled;
+                case VkFormat.R16G16B16Sscaled:
+                    return VkFormat.R16G16Sscaled;
+
+                case VkFormat.R16G16B16A16Sfloat:
+                    return VkFormat.R16G16B16Sfloat;
+                case VkFormat.R16G16B16A16Unorm:
+                    return VkFormat.R16G16B16Unorm;
+                case VkFormat.R16G16B16A16SNorm:
+                    return VkFormat.R16G16B16SNorm;
+                case VkFormat.R16G16B16A16Uint:
+                    return VkFormat.R16G16B16Uint;
+                case VkFormat.R16G16B16A16Sint:
+                    return VkFormat.R16G16B16Sint;
+                case VkFormat.R16G16B16A16Uscaled:
+                    return VkFormat.R16G16B16Uscaled;
+                case VkFormat.R16G16B16A16Sscaled:
+                    return VkFormat.R16G16B16Sscaled;
+
+                case VkFormat.R32G32Sfloat:
+                    return VkFormat.R32Sfloat;
+                case VkFormat.R32G32Uint:
+                    return VkFormat.R32Uint;
+                case VkFormat.R32G32Sint:
+                    return VkFormat.R32Sint;
+
+                case VkFormat.R32G32B32Sfloat:
+                    return VkFormat.R32G32Sfloat;
+                case VkFormat.R32G32B32Uint:
+                    return VkFormat.R32G32Uint;
+                case VkFormat.R32G32B32Sint:
+                    return VkFormat.R32G32Sint;
+
+                case VkFormat.R32G32B32A32Sfloat:
+                    return VkFormat.R32G32B32Sfloat;
+                case VkFormat.R32G32B32A32Uint:
+                    return VkFormat.R32G32B32Uint;
+                case VkFormat.R32G32B32A32Sint:
+                    return VkFormat.R32G32B32Sint;
+            }
+
+            return format;
+        }
     }
 }

src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs

@@ -28,6 +28,7 @@ namespace Ryujinx.Graphics.Vulkan
         public readonly bool SupportsSubgroupSizeControl;
         public readonly bool SupportsShaderInt8;
         public readonly bool SupportsShaderStencilExport;
+        public readonly bool SupportsShaderStorageImageMultisample;
         public readonly bool SupportsConditionalRendering;
         public readonly bool SupportsExtendedDynamicState;
         public readonly bool SupportsMultiView;
@@ -63,6 +64,7 @@ namespace Ryujinx.Graphics.Vulkan
             bool supportsSubgroupSizeControl,
             bool supportsShaderInt8,
             bool supportsShaderStencilExport,
+            bool supportsShaderStorageImageMultisample,
             bool supportsConditionalRendering,
             bool supportsExtendedDynamicState,
             bool supportsMultiView,
@@ -97,6 +99,7 @@ namespace Ryujinx.Graphics.Vulkan
             SupportsSubgroupSizeControl = supportsSubgroupSizeControl;
             SupportsShaderInt8 = supportsShaderInt8;
             SupportsShaderStencilExport = supportsShaderStencilExport;
+            SupportsShaderStorageImageMultisample = supportsShaderStorageImageMultisample;
             SupportsConditionalRendering = supportsConditionalRendering;
             SupportsExtendedDynamicState = supportsExtendedDynamicState;
             SupportsMultiView = supportsMultiView;

src/Ryujinx.Graphics.Vulkan/PipelineState.cs

@@ -1,4 +1,5 @@
-using Silk.NET.Vulkan;
+using Ryujinx.Common.Memory;
+using Silk.NET.Vulkan;
 using System;
 
 namespace Ryujinx.Graphics.Vulkan
@@ -308,6 +309,8 @@ namespace Ryujinx.Graphics.Vulkan
         public PipelineLayout PipelineLayout;
         public SpecData SpecializationData;
 
+        private Array32<VertexInputAttributeDescription> _vertexAttributeDescriptions2;
+
         public void Initialize()
         {
             Stages = new NativeArray<PipelineShaderStageCreateInfo>(Constants.MaxShaderStages);
@@ -400,7 +403,15 @@ namespace Ryujinx.Graphics.Vulkan
 
             Pipeline pipelineHandle = default;
 
+            bool isMoltenVk = gd.IsMoltenVk;
+
+            if (isMoltenVk)
+            {
+                UpdateVertexAttributeDescriptions();
+            }
+
             fixed (VertexInputAttributeDescription* pVertexAttributeDescriptions = &Internal.VertexAttributeDescriptions[0])
+            fixed (VertexInputAttributeDescription* pVertexAttributeDescriptions2 = &_vertexAttributeDescriptions2[0])
             fixed (VertexInputBindingDescription* pVertexBindingDescriptions = &Internal.VertexBindingDescriptions[0])
             fixed (Viewport* pViewports = &Internal.Viewports[0])
             fixed (Rect2D* pScissors = &Internal.Scissors[0])
@@ -410,7 +421,7 @@ namespace Ryujinx.Graphics.Vulkan
                 {
                     SType = StructureType.PipelineVertexInputStateCreateInfo,
                     VertexAttributeDescriptionCount = VertexAttributeDescriptionsCount,
-                    PVertexAttributeDescriptions = pVertexAttributeDescriptions,
+                    PVertexAttributeDescriptions = isMoltenVk ? pVertexAttributeDescriptions2 : pVertexAttributeDescriptions,
                     VertexBindingDescriptionCount = VertexBindingDescriptionsCount,
                     PVertexBindingDescriptions = pVertexBindingDescriptions
                 };
@@ -612,6 +623,40 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
+        private void UpdateVertexAttributeDescriptions()
+        {
+            // Vertex attributes exceeding the stride are invalid.
+            // In metal, they cause glitches with the vertex shader fetching incorrect values.
+            // To work around this, we reduce the format to something that doesn't exceed the stride if possible.
+            // The assumption is that the exceeding components are not actually accessed on the shader.
+
+            for (int index = 0; index < VertexAttributeDescriptionsCount; index++)
+            {
+                var attribute = Internal.VertexAttributeDescriptions[index];
+                ref var vb = ref Internal.VertexBindingDescriptions[(int)attribute.Binding];
+
+                Format format = attribute.Format;
+
+                while (vb.Stride != 0 && attribute.Offset + FormatTable.GetAttributeFormatSize(format) > vb.Stride)
+                {
+                    Format newFormat = FormatTable.DropLastComponent(format);
+
+                    if (newFormat == format)
+                    {
+                        // That case means we failed to find a format that fits within the stride,
+                        // so just restore the original format and give up.
+                        format = attribute.Format;
+                        break;
+                    }
+
+                    format = newFormat;
+                }
+
+                attribute.Format = format;
+                _vertexAttributeDescriptions2[index] = attribute;
+            }
+        }
+
         public void Dispose()
         {
             Stages.Dispose();

src/Ryujinx.Graphics.Vulkan/TextureStorage.cs

@@ -80,7 +80,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             var sampleCountFlags = ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)info.Samples);
 
-            var usage = GetImageUsageFromFormat(info.Format);
+            var usage = GetImageUsage(info.Format, info.Target, gd.Capabilities.SupportsShaderStorageImageMultisample);
 
             var flags = ImageCreateFlags.CreateMutableFormatBit;
 
@@ -293,7 +293,7 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
-        public static ImageUsageFlags GetImageUsageFromFormat(GAL.Format format)
+        public static ImageUsageFlags GetImageUsage(GAL.Format format, Target target, bool supportsMsStorage)
         {
             var usage = DefaultUsageFlags;
 
@@ -306,7 +306,7 @@ namespace Ryujinx.Graphics.Vulkan
                 usage |= ImageUsageFlags.ColorAttachmentBit;
             }
 
-            if (format.IsImageCompatible())
+            if (format.IsImageCompatible() && (supportsMsStorage || !target.IsMultisample()))
             {
                 usage |= ImageUsageFlags.StorageBit;
             }

src/Ryujinx.Graphics.Vulkan/TextureView.cs

@@ -54,7 +54,7 @@ namespace Ryujinx.Graphics.Vulkan
             gd.Textures.Add(this);
 
             var format = _gd.FormatCapabilities.ConvertToVkFormat(info.Format);
-            var usage = TextureStorage.GetImageUsageFromFormat(info.Format);
+            var usage = TextureStorage.GetImageUsage(info.Format, info.Target, gd.Capabilities.SupportsShaderStorageImageMultisample);
             var levels = (uint)info.Levels;
             var layers = (uint)info.GetLayers();
 

src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs

@@ -295,6 +295,7 @@ namespace Ryujinx.Graphics.Vulkan
                 supportsSubgroupSizeControl,
                 featuresShaderInt8.ShaderInt8,
                 _physicalDevice.IsDeviceExtensionPresent("VK_EXT_shader_stencil_export"),
+                features2.Features.ShaderStorageImageMultisample,
                 _physicalDevice.IsDeviceExtensionPresent(ExtConditionalRendering.ExtensionName),
                 _physicalDevice.IsDeviceExtensionPresent(ExtExtendedDynamicState.ExtensionName),
                 features2.Features.MultiViewport,