GPU: Rebind RTs if scale changes when binding textures (#6493)

This fixes a longstanding issue with resolution scale that could result in flickering graphics, typically the first frame something is drawn, or on camera cuts in cutscenes.

The root cause of the issue is that texture scale can be changed when binding textures or images. This typically happens because a texture becomes a view of a larger texture, such as a 400x225 texture becoming a view of a 800x450 texture with two levels. If the 400x225 texture is bound as a render target and has state [1x Undesired], but the storage texture is [2x Scaled], the render target texture's scale is changed to [2x Scaled] to match its new storage. This means the scale changed after the render target state was processed...

This can cause a number of issues. When render target state is processed, texture scales are examined and potentially changed so that they are all the same value. If one texture is scaled, all textures must be. If one texture is blacklisted from scaling, all of them must be. This results in a single resolution scale value being assigned to the TextureManager, which also scales the scissor and viewport values.

If the scale is chosen as 1x, and a later texture binding changes one of the textures to be 2x, the scale in TextureManager no longer matches all of the bound textures. What's worse, the scales in these textures could mismatch entirely. This typically results in the support buffer scale, viewport and scissor being wrong for at least one of the bound render targets.

This PR fixes the issue by re-evaluating render target state if any scale mismatches the expected scale after texture bindings happen. This can actually cause scale to change again, so it must loop back to perform texture bindings again. This can happen as many times as it needs to, but I don't expect it to happen more than once. Problematic bindings will just result in a blacklist, which will propagate to other bound targets.

src/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs

@@ -343,11 +343,22 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             bool unalignedChanged = _currentSpecState.SetHasUnalignedStorageBuffer(_channel.BufferManager.HasUnalignedStorageBuffers);
 
-            if (!_channel.TextureManager.CommitGraphicsBindings(_shaderSpecState) || unalignedChanged)
+            bool scaleMismatch;
+            do
             {
-                // Shader must be reloaded. _vtgWritesRtLayer should not change.
+                if (!_channel.TextureManager.CommitGraphicsBindings(_shaderSpecState, out scaleMismatch) || unalignedChanged)
-                UpdateShaderState();
+                {
+                    // Shader must be reloaded. _vtgWritesRtLayer should not change.
+                    UpdateShaderState();
+                }
+
+                if (scaleMismatch)
+                {
+                    // Binding textures changed scale of the bound render targets, correct the render target scale and rebind.
+                    UpdateRenderTargetState();
+                }
             }
+            while (scaleMismatch);
 
             _channel.BufferManager.CommitGraphicsBindings(_drawState.DrawIndexed);
         }

src/Ryujinx.Graphics.Gpu/Image/TextureManager.cs

@@ -360,15 +360,16 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// Commits bindings on the graphics pipeline.
         /// </summary>
         /// <param name="specState">Specialization state for the bound shader</param>
+        /// <param name="scaleMismatch">True if there is a scale mismatch in the render targets, indicating they must be re-evaluated</param>
         /// <returns>True if all bound textures match the current shader specialization state, false otherwise</returns>
-        public bool CommitGraphicsBindings(ShaderSpecializationState specState)
+        public bool CommitGraphicsBindings(ShaderSpecializationState specState, out bool scaleMismatch)
         {
             _texturePoolCache.Tick();
             _samplerPoolCache.Tick();
 
             bool result = _gpBindingsManager.CommitBindings(specState);
 
-            UpdateRenderTargets();
+            scaleMismatch = UpdateRenderTargets();
 
             return result;
         }
@@ -426,9 +427,12 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <summary>
         /// Update host framebuffer attachments based on currently bound render target buffers.
         /// </summary>
-        public void UpdateRenderTargets()
+        /// <returns>True if there is a scale mismatch in the render targets, indicating they must be re-evaluated</returns>
+        public bool UpdateRenderTargets()
         {
             bool anyChanged = false;
+            float expectedScale = RenderTargetScale;
+            bool scaleMismatch = false;
 
             Texture dsTexture = _rtDepthStencil;
             ITexture hostDsTexture = null;
@@ -448,6 +452,11 @@ namespace Ryujinx.Graphics.Gpu.Image
             {
                 _rtHostDs = hostDsTexture;
                 anyChanged = true;
+
+                if (dsTexture != null && dsTexture.ScaleFactor != expectedScale)
+                {
+                    scaleMismatch = true;
+                }
             }
 
             for (int index = 0; index < _rtColors.Length; index++)
@@ -470,6 +479,11 @@ namespace Ryujinx.Graphics.Gpu.Image
                 {
                     _rtHostColors[index] = hostTexture;
                     anyChanged = true;
+
+                    if (texture != null && texture.ScaleFactor != expectedScale)
+                    {
+                        scaleMismatch = true;
+                    }
                 }
             }
 
@@ -477,6 +491,8 @@ namespace Ryujinx.Graphics.Gpu.Image
             {
                 _context.Renderer.Pipeline.SetRenderTargets(_rtHostColors, _rtHostDs);
             }
+
+            return scaleMismatch;
         }
 
         /// <summary>

src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs

@@ -781,7 +781,9 @@ namespace Ryujinx.Graphics.Vulkan
             {
                 PrimitiveTopology.Quads => PrimitiveTopology.Triangles,
                 PrimitiveTopology.QuadStrip => PrimitiveTopology.TriangleStrip,
-                PrimitiveTopology.TriangleFan => Capabilities.PortabilitySubset.HasFlag(PortabilitySubsetFlags.NoTriangleFans) ? PrimitiveTopology.Triangles : topology,
+                PrimitiveTopology.TriangleFan or PrimitiveTopology.Polygon => Capabilities.PortabilitySubset.HasFlag(PortabilitySubsetFlags.NoTriangleFans)
+                    ? PrimitiveTopology.Triangles
+                    : topology,
                 _ => topology,
             };
         }
@@ -791,7 +793,7 @@ namespace Ryujinx.Graphics.Vulkan
             return topology switch
             {
                 PrimitiveTopology.Quads => true,
-                PrimitiveTopology.TriangleFan => Capabilities.PortabilitySubset.HasFlag(PortabilitySubsetFlags.NoTriangleFans),
+                PrimitiveTopology.TriangleFan or PrimitiveTopology.Polygon => Capabilities.PortabilitySubset.HasFlag(PortabilitySubsetFlags.NoTriangleFans),
                 _ => false,
             };
         }