GPU: Dispose Renderer after running deferred actions (#5144)

* GAL: Dispose Renderer after running deferred actions

Deferred actions from disposing physical memory instances always dispose the resources in their caches. The renderer can't be disposed before these resources get disposed, otherwise the dispose actions will not actually run, and the ThreadedRenderer may get stuck trying to enqueue too many commands when there is nothing consuming them.

This should fix most instances of the emulator freezing on close.

* Wait for main render commands to finish, but keep RenderThread alive til dispose

* Address some feedback.

* No parameterize needed

* Set thread name as part of constructor

* Port to Ava and SDL2

src/Ryujinx.Ava/AppHost.cs

@@ -92,6 +92,8 @@ namespace Ryujinx.Ava
         private bool _isActive;
         private bool _renderingStarted;
 
+        private ManualResetEvent _gpuDoneEvent;
+
         private IRenderer                        _renderer;
         private readonly Thread                  _renderingThread;
         private readonly CancellationTokenSource _gpuCancellationTokenSource;
@@ -183,6 +185,7 @@ namespace Ryujinx.Ava
             ConfigurationState.Instance.Multiplayer.LanInterfaceId.Event   += UpdateLanInterfaceIdState;
 
             _gpuCancellationTokenSource = new CancellationTokenSource();
+            _gpuDoneEvent = new ManualResetEvent(false);
         }
 
         private void TopLevel_PointerEnterOrMoved(object sender, PointerEventArgs e)
@@ -423,10 +426,10 @@ namespace Ryujinx.Ava
 
             _isActive = false;
 
-            if (_renderingThread.IsAlive)
+            // NOTE: The render loop is allowed to stay alive until the renderer itself is disposed, as it may handle resource dispose.
-            {
+            // We only need to wait for all commands submitted during the main gpu loop to be processed.
-                _renderingThread.Join();
+            _gpuDoneEvent.WaitOne();
-            }
+            _gpuDoneEvent.Dispose();
 
             DisplaySleep.Restore();
 
@@ -917,6 +920,14 @@ namespace Ryujinx.Ava
                         UpdateStatus();
                     }
                 }
+
+                // Make sure all commands in the run loop are fully executed before leaving the loop.
+                if (Device.Gpu.Renderer is ThreadedRenderer threaded)
+                {
+                    threaded.FlushThreadedCommands();
+                }
+
+                _gpuDoneEvent.Set();
             });
 
             (_rendererHost.EmbeddedWindow as EmbeddedWindowOpenGL)?.MakeCurrent(null);

src/Ryujinx.Ava/UI/Windows/AboutWindow.axaml

@@ -58,11 +58,20 @@
                         JustifyContent="SpaceAround"
                         RowSpacing="2">
                         <TextBlock
+                            HorizontalAlignment="Center"
+                            VerticalAlignment="Center"
                             FontSize="28"
                             FontWeight="Bold"
                             Text="Ryujinx"
-                            TextAlignment="Left" />
+                            TextAlignment="Center"
-                        <TextBlock Text="(REE-YOU-JINX)" TextAlignment="Left" />
+                            Width="100" />
+                        <TextBlock
+                            HorizontalAlignment="Center"
+                            VerticalAlignment="Center"
+                            FontSize="11"
+                            Text="(REE-YOU-JINX)"
+                            TextAlignment="Center"
+                            Width="100" />
                     </flex:FlexPanel>
                 </Grid>
                 <TextBlock

src/Ryujinx.Graphics.GAL/IRenderer.cs

@@ -1,5 +1,6 @@
 using Ryujinx.Common.Configuration;
 using System;
+using System.Threading;
 
 namespace Ryujinx.Graphics.GAL
 {
@@ -52,7 +53,7 @@ namespace Ryujinx.Graphics.GAL
 
         void ResetCounter(CounterType type);
 
-        void RunLoop(Action gpuLoop)
+        void RunLoop(ThreadStart gpuLoop)
         {
             gpuLoop();
         }

src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs

@@ -30,7 +30,6 @@ namespace Ryujinx.Graphics.GAL.Multithreading
         private IRenderer _baseRenderer;
         private Thread _gpuThread;
         private Thread _backendThread;
-        private bool _disposed;
         private bool _running;
 
         private AutoResetEvent _frameComplete = new AutoResetEvent(true);
@@ -98,19 +97,17 @@ namespace Ryujinx.Graphics.GAL.Multithreading
             _refQueue = new object[MaxRefsPerCommand * QueueCount];
         }
 
-        public void RunLoop(Action gpuLoop)
+        public void RunLoop(ThreadStart gpuLoop)
         {
             _running = true;
 
             _backendThread = Thread.CurrentThread;
 
-            _gpuThread = new Thread(() => {
+            _gpuThread = new Thread(gpuLoop)
-                gpuLoop();
+            {
-                _running = false;
+                Name = "GPU.MainThread"
-                _galWorkAvailable.Set();
+            };
-            });
 
-            _gpuThread.Name = "GPU.MainThread";
             _gpuThread.Start();
 
             RenderLoop();
@@ -120,7 +117,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
         {
             // Power through the render queue until the Gpu thread work is done.
 
-            while (_running && !_disposed)
+            while (_running)
             {
                 _galWorkAvailable.Wait();
                 _galWorkAvailable.Reset();
@@ -488,12 +485,23 @@ namespace Ryujinx.Graphics.GAL.Multithreading
             return _baseRenderer.PrepareHostMapping(address, size);
         }
 
+        public void FlushThreadedCommands()
+        {
+            SpinWait wait = new();
+
+            while (Volatile.Read(ref _commandCount) > 0)
+            {
+                wait.SpinOnce();
+            }
+        }
+
         public void Dispose()
         {
             // Dispose must happen from the render thread, after all commands have completed.
 
             // Stop the GPU thread.
-            _disposed = true;
+            _running = false;
+            _galWorkAvailable.Set();
 
             if (_gpuThread != null && _gpuThread.IsAlive)
             {

src/Ryujinx.Graphics.Gpu/GpuContext.cs

@@ -390,7 +390,6 @@ namespace Ryujinx.Graphics.Gpu
         /// </summary>
         public void Dispose()
         {
-            Renderer.Dispose();
             GPFifo.Dispose();
             HostInitalized.Dispose();
 
@@ -403,6 +402,8 @@ namespace Ryujinx.Graphics.Gpu
             PhysicalMemoryRegistry.Clear();
 
             RunDeferredActions();
+
+            Renderer.Dispose();
         }
     }
 }

src/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs

@@ -17,8 +17,6 @@ namespace Ryujinx.Graphics.Gpu.Shader
         private readonly ResourceCounts _resourceCounts;
         private readonly int _stageIndex;
 
-        private readonly int[] _constantBufferBindings;
-
         /// <summary>
         /// Creates a new GPU accessor.
         /// </summary>
@@ -28,12 +26,6 @@ namespace Ryujinx.Graphics.Gpu.Shader
             _context = context;
             _resourceCounts = resourceCounts;
             _stageIndex = stageIndex;
-
-            if (context.Capabilities.Api != TargetApi.Vulkan)
-            {
-                _constantBufferBindings = new int[Constants.TotalGpUniformBuffers];
-                _constantBufferBindings.AsSpan().Fill(-1);
-            }
         }
 
         public int QueryBindingConstantBuffer(int index)
@@ -45,15 +37,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
             }
             else
             {
-                int binding = _constantBufferBindings[index];
+                return _resourceCounts.UniformBuffersCount++;
-
-                if (binding < 0)
-                {
-                    binding = _resourceCounts.UniformBuffersCount++;
-                    _constantBufferBindings[index] = binding;
-                }
-
-                return binding;
             }
         }
 

src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs

@@ -17,6 +17,8 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         private readonly HashSet<int> _usedConstantBufferBindings;
 
+        public ShaderProperties Properties => _properties;
+
         public ResourceManager(ShaderStage stage, IGpuAccessor gpuAccessor, ShaderProperties properties)
         {
             _gpuAccessor = gpuAccessor;
@@ -98,19 +100,6 @@ namespace Ryujinx.Graphics.Shader.Translation
             _properties.AddConstantBuffer(binding, new BufferDefinition(BufferLayout.Std140, 0, binding, name, type));
         }
 
-        public void InheritFrom(ResourceManager other)
-        {
-            for (int i = 0; i < other._cbSlotToBindingMap.Length; i++)
-            {
-                int binding = other._cbSlotToBindingMap[i];
-
-                if (binding >= 0)
-                {
-                    _cbSlotToBindingMap[i] = binding;
-                }
-            }
-        }
-
         public static string GetShaderStagePrefix(ShaderStage stage)
         {
             uint index = (uint)stage;

src/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs

@@ -39,9 +39,9 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public TranslationOptions Options { get; }
 
-        public ShaderProperties Properties { get; }
+        public ShaderProperties Properties => ResourceManager.Properties;
 
-        public ResourceManager ResourceManager { get; }
+        public ResourceManager ResourceManager { get; set; }
 
         public bool TransformFeedbackEnabled { get; }
 
@@ -159,8 +159,7 @@ namespace Ryujinx.Graphics.Shader.Translation
             _sbSlots        = new Dictionary<int, int>();
             _sbSlotsReverse = new Dictionary<int, int>();
 
-            Properties = new ShaderProperties();
+            ResourceManager = new ResourceManager(stage, gpuAccessor, new ShaderProperties());
-            ResourceManager = new ResourceManager(stage, gpuAccessor, Properties);
         }
 
         public ShaderConfig(
@@ -429,8 +428,6 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public void InheritFrom(ShaderConfig other)
         {
-            ResourceManager.InheritFrom(other.ResourceManager);
-
             ClipDistancesWritten |= other.ClipDistancesWritten;
             UsedFeatures |= other.UsedFeatures;
 

src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs

@@ -155,6 +155,9 @@ namespace Ryujinx.Graphics.Shader.Translation
             {
                 other._config.MergeOutputUserAttributes(_config.UsedOutputAttributes, Enumerable.Empty<int>());
 
+                // We need to share the resource manager since both shaders accesses the same constant buffers.
+                other._config.ResourceManager = _config.ResourceManager;
+
                 FunctionCode[] otherCode = EmitShader(other._program, other._config, initializeOutputs: true, out int aStart);
 
                 code = Combine(otherCode, code, aStart);

src/Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -358,7 +358,7 @@ namespace Ryujinx.Graphics.Vulkan
 
         public void Draw(int vertexCount, int instanceCount, int firstVertex, int firstInstance)
         {
-            if (!_program.IsLinked)
+            if (!_program.IsLinked || vertexCount == 0)
             {
                 return;
             }
@@ -422,7 +422,7 @@ namespace Ryujinx.Graphics.Vulkan
 
         public void DrawIndexed(int indexCount, int instanceCount, int firstIndex, int firstVertex, int firstInstance)
         {
-            if (!_program.IsLinked)
+            if (!_program.IsLinked || indexCount == 0)
             {
                 return;
             }

src/Ryujinx.Headless.SDL2/Options.cs

@@ -130,7 +130,7 @@ namespace Ryujinx.Headless.SDL2
         public float AudioVolume { get; set; }
 
         [Option("use-hypervisor", Required = false, Default = true, HelpText = "Uses Hypervisor over JIT if available.")]
-        public bool UseHypervisor { get; set; }
+        public bool? UseHypervisor { get; set; }
 
         [Option("lan-interface-id", Required = false, Default = "0", HelpText = "GUID for the network interface used by LAN.")]
         public string MultiplayerLanInterfaceId { get; set; }

src/Ryujinx.Headless.SDL2/Program.cs

@@ -339,6 +339,15 @@ namespace Ryujinx.Headless.SDL2
 
             GraphicsConfig.EnableShaderCache = true;
 
+            if (OperatingSystem.IsMacOS())
+            {
+                if (option.GraphicsBackend == GraphicsBackend.OpenGl)
+                {
+                    option.GraphicsBackend = GraphicsBackend.Vulkan;
+                    Logger.Warning?.Print(LogClass.Application, "OpenGL is not supported on macOS, switching to Vulkan!");
+                }
+            }
+
             IGamepad gamepad;
 
             if (option.ListInputIds)
@@ -550,7 +559,7 @@ namespace Ryujinx.Headless.SDL2
                                                                   options.IgnoreMissingServices,
                                                                   options.AspectRatio,
                                                                   options.AudioVolume,
-                                                                  options.UseHypervisor,
+                                                                  options.UseHypervisor ?? true,
                                                                   options.MultiplayerLanInterfaceId);
 
             return new Switch(configuration);
@@ -703,4 +712,4 @@ namespace Ryujinx.Headless.SDL2
             return true;
         }
     }
-}
+}

src/Ryujinx.Headless.SDL2/Ryujinx.Headless.SDL2.csproj

@@ -7,6 +7,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Version>1.0.0-dirty</Version>
     <DefineConstants Condition=" '$(ExtraDefineConstants)' != '' ">$(DefineConstants);$(ExtraDefineConstants)</DefineConstants>
+    <SigningCertificate Condition=" '$(SigningCertificate)' == '' ">-</SigningCertificate>
     <TieredPGO>true</TieredPGO>
   </PropertyGroup>
 
@@ -15,6 +16,10 @@
     <PackageReference Include="Ryujinx.Graphics.Nvdec.Dependencies" />
   </ItemGroup>
 
+  <Target Name="PostBuild" AfterTargets="PostBuildEvent" Condition="$([MSBuild]::IsOSPlatform('OSX'))">
+    <Exec Command="codesign --entitlements '$(ProjectDir)..\..\distribution\macos\entitlements.xml' -f --deep -s $(SigningCertificate) '$(TargetDir)$(TargetName)'" />
+  </Target>
+
   <ItemGroup>
     <ProjectReference Include="..\Ryujinx.Graphics.Vulkan\Ryujinx.Graphics.Vulkan.csproj" />
     <ProjectReference Include="..\Ryujinx.Input\Ryujinx.Input.csproj" />
@@ -29,6 +34,7 @@
 
   <ItemGroup>
     <PackageReference Include="CommandLineParser" />
+    <PackageReference Include="Ryujinx.Graphics.Vulkan.Dependencies.MoltenVK" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'win10-x64'" />
   </ItemGroup>
 
   <ItemGroup>
@@ -63,4 +69,4 @@
     <PublishTrimmed>true</PublishTrimmed>
     <TrimMode>partial</TrimMode>
   </PropertyGroup>
-</Project>
+</Project>

src/Ryujinx.Headless.SDL2/WindowBase.cs

@@ -62,6 +62,7 @@ namespace Ryujinx.Headless.SDL2
         private readonly long _ticksPerFrame;
         private readonly CancellationTokenSource _gpuCancellationTokenSource;
         private readonly ManualResetEvent _exitEvent;
+        private readonly ManualResetEvent _gpuDoneEvent;
 
         private long _ticks;
         private bool _isActive;
@@ -91,6 +92,7 @@ namespace Ryujinx.Headless.SDL2
             _ticksPerFrame = Stopwatch.Frequency / TargetFps;
             _gpuCancellationTokenSource = new CancellationTokenSource();
             _exitEvent = new ManualResetEvent(false);
+            _gpuDoneEvent = new ManualResetEvent(false);
             _aspectRatio = aspectRatio;
             _enableMouse = enableMouse;
             HostUiTheme = new HeadlessHostUiTheme();
@@ -275,6 +277,14 @@ namespace Ryujinx.Headless.SDL2
                         _ticks = Math.Min(_ticks - _ticksPerFrame, _ticksPerFrame);
                     }
                 }
+
+                // Make sure all commands in the run loop are fully executed before leaving the loop.
+                if (Device.Gpu.Renderer is ThreadedRenderer threaded)
+                {
+                    threaded.FlushThreadedCommands();
+                }
+
+                _gpuDoneEvent.Set();
             });
 
             FinalizeWindowRenderer();
@@ -404,7 +414,10 @@ namespace Ryujinx.Headless.SDL2
 
             MainLoop();
 
-            renderLoopThread.Join();
+            // NOTE: The render loop is allowed to stay alive until the renderer itself is disposed, as it may handle resource dispose.
+            // We only need to wait for all commands submitted during the main gpu loop to be processed.
+            _gpuDoneEvent.WaitOne();
+            _gpuDoneEvent.Dispose();
             nvStutterWorkaround?.Join();
 
             Exit();

src/Ryujinx/Ui/RendererWidgetBase.cs

@@ -65,6 +65,7 @@ namespace Ryujinx.Ui
         private KeyboardHotkeyState _prevHotkeyState;
 
         private readonly ManualResetEvent _exitEvent;
+        private readonly ManualResetEvent _gpuDoneEvent;
 
         private readonly CancellationTokenSource _gpuCancellationTokenSource;
 
@@ -110,6 +111,7 @@ namespace Ryujinx.Ui
                           | EventMask.KeyReleaseMask));
 
             _exitEvent = new ManualResetEvent(false);
+            _gpuDoneEvent = new ManualResetEvent(false);
 
             _gpuCancellationTokenSource = new CancellationTokenSource();
 
@@ -499,6 +501,14 @@ namespace Ryujinx.Ui
                         _ticks = Math.Min(_ticks - _ticksPerFrame, _ticksPerFrame);
                     }
                 }
+
+                // Make sure all commands in the run loop are fully executed before leaving the loop.
+                if (Device.Gpu.Renderer is ThreadedRenderer threaded)
+                {
+                    threaded.FlushThreadedCommands();
+                }
+
+                _gpuDoneEvent.Set();
             });
         }
 
@@ -542,7 +552,10 @@ namespace Ryujinx.Ui
 
             MainLoop();
 
-            renderLoopThread.Join();
+            // NOTE: The render loop is allowed to stay alive until the renderer itself is disposed, as it may handle resource dispose.
+            // We only need to wait for all commands submitted during the main gpu loop to be processed.
+            _gpuDoneEvent.WaitOne();
+            _gpuDoneEvent.Dispose();
             nvStutterWorkaround?.Join();
 
             Exit();