GPU: Track buffer migrations and flush source on incomplete copy (#3952)

* Track buffer migrations and flush source on incomplete copy

Makes sure that the modified range list is always from the latest iteration of the buffer, and flushes earlier iterations of a buffer if the data has not been migrated yet.

* Cleanup 1

* Reduce cost for redundant signal checks on Vulkan

* Only inherit the range list if there are pending ranges.

* Fix OpenGL

* Address Feedback

* Whoops

README.md

@@ -36,7 +36,7 @@
 
 ## Compatibility
 
-As of October 2022, Ryujinx has been tested on approximately 3,700 titles; over 3,500 boot past menus and into gameplay, with roughly 3,000 of those being considered playable.
+As of November 2022, Ryujinx has been tested on approximately 3,800 titles; over 3,600 boot past menus and into gameplay, with roughly 3,200 of those being considered playable.
 You can check out the compatibility list [here](https://github.com/Ryujinx/Ryujinx-Games-List/issues). Anyone is free to submit a new game test or update an existing game test entry; simply follow the new issue template and testing guidelines, or post as a reply to the applicable game issue. Use the search function to see if a game has been tested already!
 
 ## Usage
@@ -90,7 +90,7 @@ Ryujinx system files are stored in the `Ryujinx` folder. This folder is located
 
 - **GPU**
 
-  The GPU emulator emulates the Switch's Maxwell GPU using either the OpenGL (version 4.5 minimum) or Vulkan APIs through a custom build of OpenTK or Silk.NET respectively. There are currently four graphics enhancements available to the end user in Ryujinx: Disk Shader Caching, Resolution Scaling, Aspect Ratio Adjustment, and Anisotropic Filtering. These enhancements can be adjusted or toggled as desired in the GUI.
+  The GPU emulator emulates the Switch's Maxwell GPU using either the OpenGL (version 4.5 minimum), Vulkan, or Metal (via MoltenVK) APIs through a custom build of OpenTK or Silk.NET respectively. There are currently four graphics enhancements available to the end user in Ryujinx: Disk Shader Caching, Resolution Scaling, Aspect Ratio Adjustment, and Anisotropic Filtering. These enhancements can be adjusted or toggled as desired in the GUI.
 
 - **Input**
 

Ryujinx.Audio.Backends.OpenAL/Ryujinx.Audio.Backends.OpenAL.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="OpenTK.OpenAL" Version="4.7.2" />
+    <PackageReference Include="OpenTK.OpenAL" Version="4.7.5" />
   </ItemGroup>
 
   <ItemGroup>

Ryujinx.Ava/Program.cs

@@ -1,8 +1,5 @@
 using ARMeilleure.Translation.PTC;
 using Avalonia;
-using Avalonia.Rendering;
-using Avalonia.Threading;
-using Ryujinx.Ava.Ui.Controls;
 using Ryujinx.Ava.Ui.Windows;
 using Ryujinx.Common;
 using Ryujinx.Common.Configuration;
@@ -23,18 +20,15 @@ namespace Ryujinx.Ava
 {
     internal class Program
     {
-        public static double      WindowScaleFactor { get; set; }
-        public static double      ActualScaleFactor { get; set; }
-        public static string      Version           { get; private set; }
-        public static string      ConfigurationPath { get; private set; }
-        public static bool        PreviewerDetached { get; private set; }
-        public static RenderTimer RenderTimer       { get; private set; }
+        public static double WindowScaleFactor { get; set; }
+        public static string Version           { get; private set; }
+        public static string ConfigurationPath { get; private set; }
+        public static bool   PreviewerDetached { get; private set; }
 
         [DllImport("user32.dll", SetLastError = true)]
         public static extern int MessageBoxA(IntPtr hWnd, string text, string caption, uint type);
 
         private const uint MB_ICONWARNING = 0x30;
-        private const int  BaseDpi        = 96;
 
         public static void Main(string[] args)
         {
@@ -49,11 +43,7 @@ namespace Ryujinx.Ava
 
             Initialize(args);
 
-            RenderTimer = new RenderTimer();
-
             BuildAvaloniaApp().StartWithClassicDesktopLifetime(args);
-
-            RenderTimer.Dispose();
         }
 
         public static AppBuilder BuildAvaloniaApp()
@@ -65,7 +55,7 @@ namespace Ryujinx.Ava
                     EnableMultiTouch = true,
                     EnableIme        = true,
                     UseEGL           = false,
-                    UseGpu           = false
+                    UseGpu           = true
                 })
                 .With(new Win32PlatformOptions
                 {
@@ -75,12 +65,6 @@ namespace Ryujinx.Ava
                     CompositionBackdropCornerRadius = 8.0f,
                 })
                 .UseSkia()
-                .AfterSetup(_ =>
-                {
-                    AvaloniaLocator.CurrentMutable
-                        .Bind<IRenderTimer>().ToConstant(RenderTimer)
-                        .Bind<IRenderLoop>().ToConstant(new RenderLoop(RenderTimer, Dispatcher.UIThread));
-                })
                 .LogToTrace();
         }
 
@@ -115,7 +99,6 @@ namespace Ryujinx.Ava
             ForceDpiAware.Windows();
 
             WindowScaleFactor = ForceDpiAware.GetWindowScaleFactor();
-            ActualScaleFactor = ForceDpiAware.GetActualScaleFactor() / BaseDpi;
 
             // Logging system information.
             PrintSystemInfo();

Ryujinx.Ava/Ryujinx.Ava.csproj

@@ -30,7 +30,7 @@
     <PackageReference Include="FluentAvaloniaUI" Version="1.4.5" />
     <PackageReference Include="XamlNameReferenceGenerator" Version="1.4.2" />
 
-    <PackageReference Include="OpenTK.Core" Version="4.7.2" />
+    <PackageReference Include="OpenTK.Core" Version="4.7.5" />
     <PackageReference Include="Ryujinx.Audio.OpenAL.Dependencies" Version="1.21.0.1" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
     <PackageReference Include="Ryujinx.Graphics.Nvdec.Dependencies" Version="5.0.1-build10" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
     <PackageReference Include="Silk.NET.Vulkan" Version="2.10.1" />

Ryujinx.Ava/Ui/Controls/RenderTimer.cs

@@ -1,100 +0,0 @@
-using Avalonia.Rendering;
-using System;
-using System.Threading;
-using System.Timers;
-
-namespace Ryujinx.Ava.Ui.Controls
-{
-    internal class RenderTimer : IRenderTimer, IDisposable
-    {
-        public event Action<TimeSpan> Tick
-        {
-            add
-            {
-                _tick += value;
-
-                if (_subscriberCount++ == 0)
-                {
-                    Start();
-                }
-            }
-
-            remove
-            {
-                if (--_subscriberCount == 0)
-                {
-                    Stop();
-                }
-
-                _tick -= value;
-            }
-        }
-
-        private Thread _tickThread;
-        private readonly System.Timers.Timer _timer;
-
-        private Action<TimeSpan> _tick;
-        private int _subscriberCount;
-
-        private bool _isRunning;
-
-        private AutoResetEvent _resetEvent;
-
-        public RenderTimer()
-        {
-            _timer = new System.Timers.Timer(15);
-            _resetEvent = new AutoResetEvent(true);
-            _timer.Elapsed += Timer_Elapsed;
-        }
-
-        private void Timer_Elapsed(object sender, ElapsedEventArgs e)
-        {
-            TickNow();
-        }
-
-        public void Start()
-        {
-            _timer.Start();
-            if (_tickThread == null)
-            {
-                _tickThread = new Thread(RunTick);
-                _tickThread.Name = "RenderTimerTickThread";
-                _tickThread.IsBackground = true;
-                _isRunning = true;
-                _tickThread.Start();
-            }
-        }
-
-        public void RunTick()
-        {
-            while (_isRunning)
-            {
-                _resetEvent.WaitOne();
-                _tick?.Invoke(TimeSpan.FromMilliseconds(Environment.TickCount));
-            }
-        }
-
-        public void TickNow()
-        {
-            lock (_timer)
-            {
-                _resetEvent.Set();
-            }
-        }
-
-        public void Stop()
-        {
-            _timer.Stop();
-        }
-
-        public void Dispose()
-        {
-            _timer.Elapsed -= Timer_Elapsed;
-            _timer.Stop();
-            _isRunning = false;
-            _resetEvent.Set();
-            _tickThread.Join();
-            _resetEvent.Dispose();
-        }
-    }
-}

Ryujinx.Ava/Ui/Controls/RendererHost.axaml

@@ -4,11 +4,4 @@
              xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
              mc:Ignorable="d" d:DesignWidth="800" d:DesignHeight="450"
              x:Class="Ryujinx.Ava.Ui.Controls.RendererHost">
-    <ContentControl
-        HorizontalAlignment="Stretch"
-        HorizontalContentAlignment="Stretch"
-        VerticalAlignment="Stretch"
-        VerticalContentAlignment="Stretch"
-        Name="View"
-        />
 </UserControl>

Ryujinx.Ava/Ui/Controls/RendererHost.axaml.cs

@@ -41,7 +41,7 @@ namespace Ryujinx.Ava.Ui.Controls
         {
             _currentWindow.WindowCreated += CurrentWindow_WindowCreated;
             _currentWindow.SizeChanged += CurrentWindow_SizeChanged;
-            View.Content = _currentWindow;
+            Content = _currentWindow;
         }
 
         public void CreateVulkan()

Ryujinx.Ava/Ui/Windows/MainWindow.axaml

@@ -13,7 +13,7 @@
     Title="Ryujinx"
     Width="1280"
     Height="785"
-    MinWidth="1024"
+    MinWidth="1092"
     MinHeight="680"
     d:DesignHeight="720"
     d:DesignWidth="1280"
@@ -57,6 +57,8 @@
                 <RowDefinition Height="Auto" />
             </Grid.RowDefinitions>
             <StackPanel
+                Name="MenuBar"
+                MinHeight="35"
                 Grid.Row="0"
                 Margin="0"
                 HorizontalAlignment="Stretch"
@@ -549,6 +551,7 @@
             <Grid
                 Name="StatusBar"
                 Grid.Row="2"
+                MinHeight="30"
                 Height="30"
                 Margin="0,0"
                 HorizontalAlignment="Stretch"

Ryujinx.Ava/Ui/Windows/MainWindow.axaml.cs

@@ -90,7 +90,9 @@ namespace Ryujinx.Ava.Ui.Windows
 
             Title = $"Ryujinx {Program.Version}";
 
-            Height /= Program.WindowScaleFactor;
+            // NOTE: Height of MenuBar and StatusBar is not usable here, since it would still be 0 at this point.
+            double barHeight = MenuBar.MinHeight + StatusBar.MinHeight;
+            Height = ((Height - barHeight) / Program.WindowScaleFactor) + barHeight;
             Width /= Program.WindowScaleFactor;
 
             if (Program.PreviewerDetached)

Ryujinx.Common/Ryujinx.Common.csproj

@@ -8,7 +8,7 @@
   <ItemGroup>
     <PackageReference Include="MsgPack.Cli" Version="1.0.1" />
     <PackageReference Include="System.Drawing.Common" Version="6.0.0" />
-    <PackageReference Include="System.Management" Version="6.0.0" />
+    <PackageReference Include="System.Management" Version="7.0.0" />
   </ItemGroup>
 
 </Project>

Ryujinx.Graphics.GAL/IRenderer.cs

@@ -29,6 +29,7 @@ namespace Ryujinx.Graphics.GAL
         ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size);
 
         Capabilities GetCapabilities();
+        ulong GetCurrentSync();
         HardwareInfo GetHardwareInfo();
 
         IProgram LoadProgramBinary(byte[] programBinary, bool hasFragmentShader, ShaderInfo info);

Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs

@@ -338,6 +338,11 @@ namespace Ryujinx.Graphics.GAL.Multithreading
             return box.Result;
         }
 
+        public ulong GetCurrentSync()
+        {
+            return _baseRenderer.GetCurrentSync();
+        }
+
         public HardwareInfo GetHardwareInfo()
         {
             return _baseRenderer.GetHardwareInfo();

Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs

@@ -37,6 +37,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         private bool _vsUsesDrawParameters;
         private bool _vtgWritesRtLayer;
         private byte _vsClipDistancesWritten;
+        private uint _vbEnableMask;
 
         private bool _prevDrawIndexed;
         private bool _prevDrawIndirect;
@@ -76,6 +77,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                     nameof(ThreedClassState.VertexBufferState),
                     nameof(ThreedClassState.VertexBufferEndAddress)),
 
+                // Must be done after vertex buffer updates.
                 new StateUpdateCallbackEntry(UpdateVertexAttribState, nameof(ThreedClassState.VertexAttribState)),
 
                 new StateUpdateCallbackEntry(UpdateBlendState,
@@ -852,12 +854,23 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         /// </summary>
         private void UpdateVertexAttribState()
         {
+            uint vbEnableMask = _vbEnableMask;
+
             Span<VertexAttribDescriptor> vertexAttribs = stackalloc VertexAttribDescriptor[Constants.TotalVertexAttribs];
 
             for (int index = 0; index < Constants.TotalVertexAttribs; index++)
             {
                 var vertexAttrib = _state.State.VertexAttribState[index];
 
+                int bufferIndex = vertexAttrib.UnpackBufferIndex();
+
+                if ((vbEnableMask & (1u << bufferIndex)) == 0)
+                {
+                    // Using a vertex buffer that doesn't exist is invalid, so let's use a dummy attribute for those cases.
+                    vertexAttribs[index] = new VertexAttribDescriptor(0, 0, true, Format.R32G32B32A32Float);
+                    continue;
+                }
+
                 if (!FormatTable.TryGetAttribFormat(vertexAttrib.UnpackFormat(), out Format format))
                 {
                     Logger.Debug?.Print(LogClass.Gpu, $"Invalid attribute format 0x{vertexAttrib.UnpackFormat():X}.");
@@ -866,7 +879,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 }
 
                 vertexAttribs[index] = new VertexAttribDescriptor(
-                    vertexAttrib.UnpackBufferIndex(),
+                    bufferIndex,
                     vertexAttrib.UnpackOffset(),
                     vertexAttrib.UnpackIsConstant(),
                     format);
@@ -954,6 +967,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             bool drawIndexed = _drawState.DrawIndexed;
             bool drawIndirect = _drawState.DrawIndirect;
+            uint vbEnableMask = 0;
 
             for (int index = 0; index < Constants.TotalVertexBuffers; index++)
             {
@@ -971,6 +985,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
                 ulong address = vertexBuffer.Address.Pack();
 
+                if (_channel.MemoryManager.IsMapped(address))
+                {
+                    vbEnableMask |= 1u << index;
+                }
+
                 int stride = vertexBuffer.UnpackStride();
 
                 bool instanced = _state.State.VertexBufferInstanced[index];
@@ -1017,6 +1036,12 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 _pipeline.VertexBuffers[index] = new BufferPipelineDescriptor(_channel.MemoryManager.IsMapped(address), stride, divisor);
                 _channel.BufferManager.SetVertexBuffer(index, address, size, stride, divisor);
             }
+
+            if (_vbEnableMask != vbEnableMask)
+            {
+                _vbEnableMask = vbEnableMask;
+                UpdateVertexAttribState();
+            }
         }
 
         /// <summary>

Ryujinx.Graphics.Gpu/GpuContext.cs

@@ -69,6 +69,12 @@ namespace Ryujinx.Graphics.Gpu
         /// </summary>
         internal List<Action> SyncpointActions { get; }
 
+        /// <summary>
+        /// Buffer migrations that are currently in-flight. These are checked whenever sync is created to determine if buffer migration
+        /// copies have completed on the GPU, and their data can be freed.
+        /// </summary>
+        internal List<BufferMigration> BufferMigrations { get; }
+
         /// <summary>
         /// Queue with deferred actions that must run on the render thread.
         /// </summary>
@@ -90,6 +96,7 @@ namespace Ryujinx.Graphics.Gpu
         public event Action<ShaderCacheState, int, int> ShaderCacheStateChanged;
 
         private Thread _gpuThread;
+        private bool _pendingSync;
 
         /// <summary>
         /// Creates a new instance of the GPU emulation context.
@@ -109,6 +116,7 @@ namespace Ryujinx.Graphics.Gpu
 
             SyncActions = new List<Action>();
             SyncpointActions = new List<Action>();
+            BufferMigrations = new List<BufferMigration>();
 
             DeferredActions = new Queue<Action>();
 
@@ -273,6 +281,17 @@ namespace Ryujinx.Graphics.Gpu
             SequenceNumber++;
         }
 
+        /// <summary>
+        /// Registers a buffer migration. These are checked to see if they can be disposed when the sync number increases,
+        /// and the migration copy has completed.
+        /// </summary>
+        /// <param name="migration">The buffer migration</param>
+        internal void RegisterBufferMigration(BufferMigration migration)
+        {
+            BufferMigrations.Add(migration);
+            _pendingSync = true;
+        }
+
         /// <summary>
         /// Registers an action to be performed the next time a syncpoint is incremented.
         /// This will also ensure a host sync object is created, and <see cref="SyncNumber"/> is incremented.
@@ -288,6 +307,7 @@ namespace Ryujinx.Graphics.Gpu
             else
             {
                 SyncActions.Add(action);
+                _pendingSync = true;
             }
         }
 
@@ -298,7 +318,24 @@ namespace Ryujinx.Graphics.Gpu
         /// <param name="syncpoint">True if host sync is being created by a syncpoint</param>
         public void CreateHostSyncIfNeeded(bool syncpoint)
         {
-            if (SyncActions.Count > 0 || (syncpoint && SyncpointActions.Count > 0))
+            if (BufferMigrations.Count > 0)
+            {
+                ulong currentSyncNumber = Renderer.GetCurrentSync();
+
+                for (int i = 0; i < BufferMigrations.Count; i++)
+                {
+                    BufferMigration migration = BufferMigrations[i];
+                    long diff = (long)(currentSyncNumber - migration.SyncNumber);
+
+                    if (diff >= 0)
+                    {
+                        migration.Dispose();
+                        BufferMigrations.RemoveAt(i--);
+                    }
+                }
+            }
+
+            if (_pendingSync || (syncpoint && SyncpointActions.Count > 0))
             {
                 Renderer.CreateSync(SyncNumber);
 
@@ -317,6 +354,8 @@ namespace Ryujinx.Graphics.Gpu
                 SyncActions.Clear();
                 SyncpointActions.Clear();
             }
+
+            _pendingSync = false;
         }
 
         /// <summary>

Ryujinx.Graphics.Gpu/Memory/Buffer.cs

@@ -65,6 +65,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
         private bool _useGranular;
         private bool _syncActionRegistered;
 
+        private int _referenceCount = 1;
+
         /// <summary>
         /// Creates a new instance of the buffer.
         /// </summary>
@@ -229,7 +231,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         {
             if (_modifiedRanges == null)
             {
-                _modifiedRanges = new BufferModifiedRangeList(_context);
+                _modifiedRanges = new BufferModifiedRangeList(_context, this, Flush);
             }
         }
 
@@ -290,7 +292,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="from">The buffer to inherit from</param>
         public void InheritModifiedRanges(Buffer from)
         {
-            if (from._modifiedRanges != null)
+            if (from._modifiedRanges != null && from._modifiedRanges.HasRanges)
             {
                 if (from._syncActionRegistered && !_syncActionRegistered)
                 {
@@ -310,17 +312,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
                     }
                 };
 
-                if (_modifiedRanges == null)
-                {
-                    _modifiedRanges = from._modifiedRanges;
-                    _modifiedRanges.ReregisterRanges(registerRangeAction);
+                EnsureRangeList();
 
-                    from._modifiedRanges = null;
-                }
-                else
-                {
-                    _modifiedRanges.InheritRanges(from._modifiedRanges, registerRangeAction);
-                }
+                _modifiedRanges.InheritRanges(from._modifiedRanges, registerRangeAction);
             }
         }
 
@@ -456,7 +450,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 if (ranges != null)
                 {
                     (address, size) = PageAlign(address, size);
-                    ranges.WaitForAndGetRanges(address, size, Flush);
+                    ranges.WaitForAndFlushRanges(address, size);
                 }
             }, true);
         }
@@ -508,6 +502,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
             UnmappedSequence++;
         }
 
+        /// <summary>
+        /// Increments the buffer reference count.
+        /// </summary>
+        public void IncrementReferenceCount()
+        {
+            _referenceCount++;
+        }
+
+        /// <summary>
+        /// Decrements the buffer reference count.
+        /// </summary>
+        public void DecrementReferenceCount()
+        {
+            if (--_referenceCount == 0)
+            {
+                DisposeData();
+            }
+        }
+
         /// <summary>
         /// Disposes the host buffer's data, not its tracking handles.
         /// </summary>
@@ -528,7 +541,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
             _memoryTrackingGranular?.Dispose();
             _memoryTracking?.Dispose();
 
-            DisposeData();
+            DecrementReferenceCount();
         }
     }
 }

Ryujinx.Graphics.Gpu/Memory/BufferCache.cs

@@ -273,7 +273,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
                         buffer.CopyTo(newBuffer, dstOffset);
                         newBuffer.InheritModifiedRanges(buffer);
 
-                        buffer.DisposeData();
+                        buffer.DecrementReferenceCount();
                     }
 
                     newBuffer.SynchronizeMemory(address, newSize);

Ryujinx.Graphics.Gpu/Memory/BufferMigration.cs

@@ -0,0 +1,125 @@
+using System;
+
+namespace Ryujinx.Graphics.Gpu.Memory
+{
+    /// <summary>
+    /// A record of when buffer data was copied from one buffer to another, along with the SyncNumber when the migration will be complete.
+    /// Keeps the source buffer alive for data flushes until the migration is complete.
+    /// </summary>
+    internal class BufferMigration : IDisposable
+    {
+        /// <summary>
+        /// The offset for the migrated region.
+        /// </summary>
+        private readonly ulong _offset;
+
+        /// <summary>
+        /// The size for the migrated region.
+        /// </summary>
+        private readonly ulong _size;
+
+        /// <summary>
+        /// The buffer that was migrated from.
+        /// </summary>
+        private readonly Buffer _buffer;
+
+        /// <summary>
+        /// The source range action, to be called on overlap with an unreached sync number.
+        /// </summary>
+        private readonly Action<ulong, ulong> _sourceRangeAction;
+
+        /// <summary>
+        /// The source range list.
+        /// </summary>
+        private readonly BufferModifiedRangeList _source;
+
+        /// <summary>
+        /// The destination range list. This range list must be updated when flushing the source.
+        /// </summary>
+        public readonly BufferModifiedRangeList Destination;
+
+        /// <summary>
+        /// The sync number that needs to be reached for this migration to be removed. This is set to the pending sync number on creation.
+        /// </summary>
+        public readonly ulong SyncNumber;
+
+        /// <summary>
+        /// Creates a record for a buffer migration.
+        /// </summary>
+        /// <param name="buffer">The source buffer for this migration</param>
+        /// <param name="sourceRangeAction">The flush action for the source buffer</param>
+        /// <param name="source">The modified range list for the source buffer</param>
+        /// <param name="dest">The modified range list for the destination buffer</param>
+        /// <param name="syncNumber">The sync number for when the migration is complete</param>
+        public BufferMigration(
+            Buffer buffer,
+            Action<ulong, ulong> sourceRangeAction,
+            BufferModifiedRangeList source,
+            BufferModifiedRangeList dest,
+            ulong syncNumber)
+        {
+            _offset = buffer.Address;
+            _size = buffer.Size;
+            _buffer = buffer;
+            _sourceRangeAction = sourceRangeAction;
+            _source = source;
+            Destination = dest;
+            SyncNumber = syncNumber;
+        }
+
+        /// <summary>
+        /// Determine if the given range overlaps this migration, and has not been completed yet.
+        /// </summary>
+        /// <param name="offset">Start offset</param>
+        /// <param name="size">Range size</param>
+        /// <param name="syncNumber">The sync number that was waited on</param>
+        /// <returns>True if overlapping and in progress, false otherwise</returns>
+        public bool Overlaps(ulong offset, ulong size, ulong syncNumber)
+        {
+            ulong end = offset + size;
+            ulong destEnd = _offset + _size;
+            long syncDiff = (long)(syncNumber - SyncNumber); // syncNumber is less if the copy has not completed.
+
+            return !(end <= _offset || offset >= destEnd) && syncDiff < 0;
+        }
+
+        /// <summary>
+        /// Determine if the given range matches this migration.
+        /// </summary>
+        /// <param name="offset">Start offset</param>
+        /// <param name="size">Range size</param>
+        /// <returns>True if the range exactly matches, false otherwise</returns>
+        public bool FullyMatches(ulong offset, ulong size)
+        {
+            return _offset == offset && _size == size;
+        }
+
+        /// <summary>
+        /// Perform the migration source's range action on the range provided, clamped to the bounds of the source buffer.
+        /// </summary>
+        /// <param name="offset">Start offset</param>
+        /// <param name="size">Range size</param>
+        /// <param name="syncNumber">Current sync number</param>
+        /// <param name="parent">The modified range list that originally owned this range</param>
+        public void RangeActionWithMigration(ulong offset, ulong size, ulong syncNumber, BufferModifiedRangeList parent)
+        {
+            ulong end = offset + size;
+            end = Math.Min(_offset + _size, end);
+            offset = Math.Max(_offset, offset);
+
+            size = end - offset;
+
+            _source.RangeActionWithMigration(offset, size, syncNumber, parent, _sourceRangeAction);
+        }
+
+        /// <summary>
+        /// Removes this reference to the range list, potentially allowing for the source buffer to be disposed.
+        /// </summary>
+        public void Dispose()
+        {
+            Destination.RemoveMigration(this);
+
+            _buffer.DecrementReferenceCount();
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs

@@ -1,6 +1,8 @@
-using Ryujinx.Common.Pools;
+using Ryujinx.Common.Logging;
+using Ryujinx.Common.Pools;
 using Ryujinx.Memory.Range;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace Ryujinx.Graphics.Gpu.Memory
@@ -30,17 +32,24 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </summary>
         public ulong SyncNumber { get; internal set; }
 
+        /// <summary>
+        /// The range list that originally owned this range.
+        /// </summary>
+        public BufferModifiedRangeList Parent { get; internal set; }
+
         /// <summary>
         /// Creates a new instance of a modified range.
         /// </summary>
         /// <param name="address">Start address of the range</param>
         /// <param name="size">Size of the range in bytes</param>
         /// <param name="syncNumber">The GPU sync number at the time of creation</param>
-        public BufferModifiedRange(ulong address, ulong size, ulong syncNumber)
+        /// <param name="parent">The range list that owns this range</param>
+        public BufferModifiedRange(ulong address, ulong size, ulong syncNumber, BufferModifiedRangeList parent)
         {
             Address = address;
             Size = size;
             SyncNumber = syncNumber;
+            Parent = parent;
         }
 
         /// <summary>
@@ -63,16 +72,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
         private const int BackingInitialSize = 8;
 
         private GpuContext _context;
+        private Buffer _parent;
+        private Action<ulong, ulong> _flushAction;
+
+        private List<BufferMigration> _sources;
+        private BufferMigration _migrationTarget;
 
         private object _lock = new object();
 
+        /// <summary>
+        /// Whether the modified range list has any entries or not.
+        /// </summary>
+        public bool HasRanges
+        {
+            get
+            {
+                lock (_lock)
+                {
+                    return Count > 0;
+                }
+            }
+        }
+
         /// <summary>
         /// Creates a new instance of a modified range list.
         /// </summary>
         /// <param name="context">GPU context that the buffer range list belongs to</param>
-        public BufferModifiedRangeList(GpuContext context) : base(BackingInitialSize)
+        /// <param name="parent">The parent buffer that owns this range list</param>
+        /// <param name="flushAction">The flush action for the parent buffer</param>
+        public BufferModifiedRangeList(GpuContext context, Buffer parent, Action<ulong, ulong> flushAction) : base(BackingInitialSize)
         {
             _context = context;
+            _parent = parent;
+            _flushAction = flushAction;
         }
 
         /// <summary>
@@ -142,6 +174,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
                     {
                         // Region already exists. Just update the existing sync number.
                         overlap.SyncNumber = syncNumber;
+                        overlap.Parent = this;
 
                         return;
                     }
@@ -152,18 +185,18 @@ namespace Ryujinx.Graphics.Gpu.Memory
                     {
                         // A split item must be created behind this overlap.
 
-                        Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber));
+                        Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent));
                     }
 
                     if (overlap.Address < endAddress && overlap.EndAddress > endAddress)
                     {
                         // A split item must be created after this overlap.
 
-                        Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber));
+                        Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent));
                     }
                 }
 
-                Add(new BufferModifiedRange(address, size, syncNumber));
+                Add(new BufferModifiedRange(address, size, syncNumber, this));
             }
         }
 
@@ -207,9 +240,102 @@ namespace Ryujinx.Graphics.Gpu.Memory
             }
         }
 
+        /// <summary>
+        /// Performs the given range action, or one from a migration that overlaps and has not synced yet.
+        /// </summary>
+        /// <param name="offset">The offset to pass to the action</param>
+        /// <param name="size">The size to pass to the action</param>
+        /// <param name="syncNumber">The sync number that has been reached</param>
+        /// <param name="parent">The modified range list that originally owned this range</param>
+        /// <param name="rangeAction">The action to perform</param>
+        public void RangeActionWithMigration(ulong offset, ulong size, ulong syncNumber, BufferModifiedRangeList parent, Action<ulong, ulong> rangeAction)
+        {
+            bool firstSource = true;
+
+            if (parent != this)
+            {
+                lock (_lock)
+                {
+                    if (_sources != null)
+                    {
+                        foreach (BufferMigration source in _sources)
+                        {
+                            if (source.Overlaps(offset, size, syncNumber))
+                            {
+                                if (firstSource && !source.FullyMatches(offset, size))
+                                {
+                                    // Perform this buffer's action first. The migrations will run after.
+                                    rangeAction(offset, size);
+                                }
+
+                                source.RangeActionWithMigration(offset, size, syncNumber, parent);
+
+                                firstSource = false;
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (firstSource)
+            {
+                // No overlapping migrations, or they are not meant for this range, flush the data using the given action.
+                rangeAction(offset, size);
+            }
+        }
+
+        /// <summary>
+        /// Removes modified ranges ready by the sync number from the list, and flushes their buffer data within a given address range.
+        /// </summary>
+        /// <param name="overlaps">Overlapping ranges to check</param>
+        /// <param name="rangeCount">Number of overlapping ranges</param>
+        /// <param name="highestDiff">The highest difference between an overlapping range's sync number and the current one</param>
+        /// <param name="currentSync">The current sync number</param>
+        /// <param name="address">The start address of the flush range</param>
+        /// <param name="endAddress">The end address of the flush range</param>
+        private void RemoveRangesAndFlush(
+            BufferModifiedRange[] overlaps,
+            int rangeCount,
+            long highestDiff,
+            ulong currentSync,
+            ulong address,
+            ulong endAddress)
+        {
+            lock (_lock)
+            {
+                if (_migrationTarget == null)
+                {
+                    ulong waitSync = currentSync + (ulong)highestDiff;
+
+                    for (int i = 0; i < rangeCount; i++)
+                    {
+                        BufferModifiedRange overlap = overlaps[i];
+
+                        long diff = (long)(overlap.SyncNumber - currentSync);
+
+                        if (diff <= highestDiff)
+                        {
+                            ulong clampAddress = Math.Max(address, overlap.Address);
+                            ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
+
+                            ClearPart(overlap, clampAddress, clampEnd);
+
+                            RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, overlap.Parent, _flushAction);
+                        }
+                    }
+
+                    return;
+                }
+            }
+
+            // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine.
+
+            _migrationTarget.Destination.RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress);
+        }
+
         /// <summary>
         /// Gets modified ranges within the specified region, waits on ones from a previous sync number,
-        /// and then fires the given action for each range individually.
+        /// and then fires the flush action for each range individually.
         /// </summary>
         /// <remarks>
         /// This function assumes it is called from the background thread.
@@ -218,8 +344,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </remarks>
         /// <param name="address">Start address to query</param>
         /// <param name="size">Size to query</param>
-        /// <param name="rangeAction">The action to call for each modified range</param>
-        public void WaitForAndGetRanges(ulong address, ulong size, Action<ulong, ulong> rangeAction)
+        public void WaitForAndFlushRanges(ulong address, ulong size)
         {
             ulong endAddress = address + size;
             ulong currentSync = _context.SyncNumber;
@@ -231,10 +356,23 @@ namespace Ryujinx.Graphics.Gpu.Memory
             // Range list must be consistent for this operation
             lock (_lock)
             {
-                rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps);
+                if (_migrationTarget != null)
+                {
+                    rangeCount = -1;
+                }
+                else
+                {
+                    rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps);
+                }
             }
 
-            if (rangeCount == 0)
+            if (rangeCount == -1)
+            {
+                _migrationTarget.Destination.WaitForAndFlushRanges(address, size);
+
+                return;
+            }
+            else if (rangeCount == 0)
             {
                 return;
             }
@@ -264,47 +402,37 @@ namespace Ryujinx.Graphics.Gpu.Memory
             // Wait for the syncpoint.
             _context.Renderer.WaitSync(currentSync + (ulong)highestDiff);
 
-            // Flush and remove all regions with the older syncpoint.
-            lock (_lock)
-            {
-                for (int i = 0; i < rangeCount; i++)
-                {
-                    BufferModifiedRange overlap = overlaps[i];
-
-                    long diff = (long)(overlap.SyncNumber - currentSync);
-
-                    if (diff <= highestDiff)
-                    {
-                        ulong clampAddress = Math.Max(address, overlap.Address);
-                        ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
-
-                        ClearPart(overlap, clampAddress, clampEnd);
-
-                        rangeAction(clampAddress, clampEnd - clampAddress);
-                    }
-                }
-            }
+            RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress);
         }
 
         /// <summary>
         /// Inherit ranges from another modified range list.
         /// </summary>
         /// <param name="ranges">The range list to inherit from</param>
-        /// <param name="rangeAction">The action to call for each modified range</param>
-        public void InheritRanges(BufferModifiedRangeList ranges, Action<ulong, ulong> rangeAction)
+        /// <param name="registerRangeAction">The action to call for each modified range</param>
+        public void InheritRanges(BufferModifiedRangeList ranges, Action<ulong, ulong> registerRangeAction)
         {
             BufferModifiedRange[] inheritRanges;
 
             lock (ranges._lock)
             {
-                inheritRanges = ranges.ToArray();
-            }
+                BufferMigration migration = new(ranges._parent, ranges._flushAction, ranges, this, _context.SyncNumber);
 
-            lock (_lock)
-            {
-                foreach (BufferModifiedRange range in inheritRanges)
+                ranges._parent.IncrementReferenceCount();
+                ranges._migrationTarget = migration;
+
+                _context.RegisterBufferMigration(migration);
+
+                inheritRanges = ranges.ToArray();
+
+                lock (_lock)
                 {
-                    Add(range);
+                    (_sources ??= new List<BufferMigration>()).Add(migration);
+
+                    foreach (BufferModifiedRange range in inheritRanges)
+                    {
+                        Add(range);
+                    }
                 }
             }
 
@@ -313,44 +441,20 @@ namespace Ryujinx.Graphics.Gpu.Memory
             {
                 if (range.SyncNumber != currentSync)
                 {
-                    rangeAction(range.Address, range.Size);
+                    registerRangeAction(range.Address, range.Size);
                 }
             }
         }
 
         /// <summary>
-        /// Calls the given action for modified ranges that aren't from the current sync number.
+        /// Removes a source buffer migration, indicating its copy has completed.
         /// </summary>
-        /// <param name="rangeAction">The action to call for each modified range</param>
-        public void ReregisterRanges(Action<ulong, ulong> rangeAction)
+        /// <param name="migration">The migration to remove</param>
+        public void RemoveMigration(BufferMigration migration)
         {
-            ref var ranges = ref ThreadStaticArray<BufferModifiedRange>.Get();
-            int count;
-
-            // Range list must be consistent for this operation.
             lock (_lock)
             {
-                count = Count;
-                if (ranges.Length < count)
-                {
-                    Array.Resize(ref ranges, count);
-                }
-
-                int i = 0;
-                foreach (BufferModifiedRange range in this)
-                {
-                    ranges[i++] = range;
-                }
-            }
-
-            ulong currentSync = _context.SyncNumber;
-            for (int i = 0; i < count; i++)
-            {
-                BufferModifiedRange range = ranges[i];
-                if (range.SyncNumber != currentSync)
-                {
-                    rangeAction(range.Address, range.Size);
-                }
+                _sources.Remove(migration);
             }
         }
 
@@ -362,12 +466,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
             if (overlap.Address < address)
             {
-                Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber));
+                Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent));
             }
 
             if (overlap.EndAddress > endAddress)
             {
-                Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber));
+                Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent));
             }
         }
 

Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs

@@ -22,7 +22,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
         private const ushort FileFormatVersionMajor = 1;
         private const ushort FileFormatVersionMinor = 2;
         private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor;
-        private const uint CodeGenVersion = 3897;
+        private const uint CodeGenVersion = 3943;
 
         private const string SharedTocFileName = "shared.toc";
         private const string SharedDataFileName = "shared.data";

Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs

@@ -238,6 +238,11 @@ namespace Ryujinx.Graphics.OpenGL
             _sync.Wait(id);
         }
 
+        public ulong GetCurrentSync()
+        {
+            return _sync.GetCurrent();
+        }
+
         public void Screenshot()
         {
             _window.ScreenCaptureRequested = true;

Ryujinx.Graphics.OpenGL/Ryujinx.Graphics.OpenGL.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="OpenTK.Graphics" Version="4.7.2" />
+    <PackageReference Include="OpenTK.Graphics" Version="4.7.5" />
   </ItemGroup>
 
   <ItemGroup>

Ryujinx.Graphics.OpenGL/Sync.cs

@@ -40,6 +40,37 @@ namespace Ryujinx.Graphics.OpenGL
             }
         }
 
+        public ulong GetCurrent()
+        {
+            lock (_handles)
+            {
+                ulong lastHandle = _firstHandle;
+
+                foreach (SyncHandle handle in _handles)
+                {
+                    lock (handle)
+                    {
+                        if (handle.Handle == IntPtr.Zero)
+                        {
+                            continue;
+                        }
+
+                        if (handle.ID > lastHandle)
+                        {
+                            WaitSyncStatus syncResult = GL.ClientWaitSync(handle.Handle, _syncFlags, 0);
+
+                            if (syncResult == WaitSyncStatus.AlreadySignaled)
+                            {
+                                lastHandle = handle.ID;
+                            }
+                        }
+                    }
+                }
+
+                return lastHandle;
+            }
+        }
+
         public void Wait(ulong id)
         {
             SyncHandle result = null;

Ryujinx.Graphics.OpenGL/VertexArray.cs

@@ -10,13 +10,9 @@ namespace Ryujinx.Graphics.OpenGL
     {
         public int Handle { get; private set; }
 
-        private bool _needsAttribsUpdate;
-
         private readonly VertexAttribDescriptor[] _vertexAttribs;
         private readonly VertexBufferDescriptor[] _vertexBuffers;
 
-        private int _vertexAttribsCount;
-        private int _vertexBuffersCount;
         private int _minVertexCount;
 
         private uint _vertexAttribsInUse;
@@ -76,9 +72,7 @@ namespace Ryujinx.Graphics.OpenGL
                 _vertexBuffers[bindingIndex] = vb;
             }
 
-            _vertexBuffersCount = bindingIndex;
             _minVertexCount = minVertexCount;
-            _needsAttribsUpdate = true;
         }
 
         public void SetVertexAttributes(ReadOnlySpan<VertexAttribDescriptor> vertexAttribs)
@@ -131,8 +125,6 @@ namespace Ryujinx.Graphics.OpenGL
                 _vertexAttribs[index] = attrib;
             }
 
-            _vertexAttribsCount = index;
-
             for (; index < Constants.MaxVertexAttribs; index++)
             {
                 DisableVertexAttrib(index);
@@ -160,13 +152,11 @@ namespace Ryujinx.Graphics.OpenGL
         public void PreDraw(int vertexCount)
         {
             LimitVertexBuffers(vertexCount);
-            Validate();
         }
 
         public void PreDrawVbUnbounded()
         {
             UnlimitVertexBuffers();
-            Validate();
         }
 
         public void LimitVertexBuffers(int vertexCount)
@@ -252,36 +242,6 @@ namespace Ryujinx.Graphics.OpenGL
             _vertexBuffersLimited = 0;
         }
 
-        public void Validate()
-        {
-            for (int attribIndex = 0; attribIndex < _vertexAttribsCount; attribIndex++)
-            {
-                VertexAttribDescriptor attrib = _vertexAttribs[attribIndex];
-
-                if (!attrib.IsZero)
-                {
-                    if ((uint)attrib.BufferIndex >= _vertexBuffersCount)
-                    {
-                        DisableVertexAttrib(attribIndex);
-                        continue;
-                    }
-
-                    if (_vertexBuffers[attrib.BufferIndex].Buffer.Handle == BufferHandle.Null)
-                    {
-                        DisableVertexAttrib(attribIndex);
-                        continue;
-                    }
-
-                    if (_needsAttribsUpdate)
-                    {
-                        EnableVertexAttrib(attribIndex);
-                    }
-                }
-            }
-
-            _needsAttribsUpdate = false;
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private void EnableVertexAttrib(int index)
         {

Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs

@@ -234,7 +234,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             var source = operation.GetSource(0);
 
             var uvec4Type = context.TypeVector(context.TypeU32(), 4);
-            var execution = context.Constant(context.TypeU32(), 3); // Subgroup
+            var execution = context.Constant(context.TypeU32(), Scope.Subgroup);
 
             var maskVector = context.GroupNonUniformBallot(uvec4Type, execution, context.Get(AggregateType.Bool, source));
             var mask = context.CompositeExtract(context.TypeU32(), maskVector, (SpvLiteralInteger)0);
@@ -1233,7 +1233,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             var maxThreadId = context.BitwiseOr(context.TypeU32(), minThreadId, clampNotSegMask);
             var srcThreadId = context.BitwiseOr(context.TypeU32(), indexNotSegMask, minThreadId);
             var valid = context.ULessThanEqual(context.TypeBool(), srcThreadId, maxThreadId);
-            var value = context.SubgroupReadInvocationKHR(context.TypeFP32(), x, srcThreadId);
+            var value = context.GroupNonUniformShuffle(context.TypeFP32(), context.Constant(context.TypeU32(), (int)Scope.Subgroup), x, srcThreadId);
             var result = context.Select(context.TypeFP32(), valid, value, x);
 
             var validLocal = (AstOperand)operation.GetSource(3);
@@ -1263,7 +1263,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             var maxThreadId = context.BitwiseOr(context.TypeU32(), minThreadId, clampNotSegMask);
             var srcThreadId = context.IAdd(context.TypeU32(), threadId, index);
             var valid = context.ULessThanEqual(context.TypeBool(), srcThreadId, maxThreadId);
-            var value = context.SubgroupReadInvocationKHR(context.TypeFP32(), x, srcThreadId);
+            var value = context.GroupNonUniformShuffle(context.TypeFP32(), context.Constant(context.TypeU32(), (int)Scope.Subgroup), x, srcThreadId);
             var result = context.Select(context.TypeFP32(), valid, value, x);
 
             var validLocal = (AstOperand)operation.GetSource(3);
@@ -1289,7 +1289,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             var minThreadId = context.BitwiseAnd(context.TypeU32(), threadId, segMask);
             var srcThreadId = context.ISub(context.TypeU32(), threadId, index);
             var valid = context.SGreaterThanEqual(context.TypeBool(), srcThreadId, minThreadId);
-            var value = context.SubgroupReadInvocationKHR(context.TypeFP32(), x, srcThreadId);
+            var value = context.GroupNonUniformShuffle(context.TypeFP32(), context.Constant(context.TypeU32(), (int)Scope.Subgroup), x, srcThreadId);
             var result = context.Select(context.TypeFP32(), valid, value, x);
 
             var validLocal = (AstOperand)operation.GetSource(3);
@@ -1319,7 +1319,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             var maxThreadId = context.BitwiseOr(context.TypeU32(), minThreadId, clampNotSegMask);
             var srcThreadId = context.BitwiseXor(context.TypeU32(), threadId, index);
             var valid = context.ULessThanEqual(context.TypeBool(), srcThreadId, maxThreadId);
-            var value = context.SubgroupReadInvocationKHR(context.TypeFP32(), x, srcThreadId);
+            var value = context.GroupNonUniformShuffle(context.TypeFP32(), context.Constant(context.TypeU32(), (int)Scope.Subgroup), x, srcThreadId);
             var result = context.Select(context.TypeFP32(), valid, value, x);
 
             var validLocal = (AstOperand)operation.GetSource(3);
@@ -1861,19 +1861,22 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
 
         private static OperationResult GenerateVoteAll(CodeGenContext context, AstOperation operation)
         {
-            var result = context.SubgroupAllKHR(context.TypeBool(), context.Get(AggregateType.Bool, operation.GetSource(0)));
+            var execution = context.Constant(context.TypeU32(), Scope.Subgroup);
+            var result = context.GroupNonUniformAll(context.TypeBool(), execution, context.Get(AggregateType.Bool, operation.GetSource(0)));
             return new OperationResult(AggregateType.Bool, result);
         }
 
         private static OperationResult GenerateVoteAllEqual(CodeGenContext context, AstOperation operation)
         {
-            var result = context.SubgroupAllEqualKHR(context.TypeBool(), context.Get(AggregateType.Bool, operation.GetSource(0)));
+            var execution = context.Constant(context.TypeU32(), Scope.Subgroup);
+            var result = context.GroupNonUniformAllEqual(context.TypeBool(), execution, context.Get(AggregateType.Bool, operation.GetSource(0)));
             return new OperationResult(AggregateType.Bool, result);
         }
 
         private static OperationResult GenerateVoteAny(CodeGenContext context, AstOperation operation)
         {
-            var result = context.SubgroupAnyKHR(context.TypeBool(), context.Get(AggregateType.Bool, operation.GetSource(0)));
+            var execution = context.Constant(context.TypeU32(), Scope.Subgroup);
+            var result = context.GroupNonUniformAny(context.TypeBool(), execution, context.Get(AggregateType.Bool, operation.GetSource(0)));
             return new OperationResult(AggregateType.Bool, result);
         }
 

Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs

@@ -50,12 +50,12 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             CodeGenContext context = new CodeGenContext(info, config, instPool, integerPool);
 
             context.AddCapability(Capability.GroupNonUniformBallot);
+            context.AddCapability(Capability.GroupNonUniformShuffle);
+            context.AddCapability(Capability.GroupNonUniformVote);
             context.AddCapability(Capability.ImageBuffer);
             context.AddCapability(Capability.ImageGatherExtended);
             context.AddCapability(Capability.ImageQuery);
             context.AddCapability(Capability.SampledBuffer);
-            context.AddCapability(Capability.SubgroupBallotKHR);
-            context.AddCapability(Capability.SubgroupVoteKHR);
 
             if (config.TransformFeedbackEnabled && config.LastInVertexPipeline)
             {
@@ -94,9 +94,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                 context.AddCapability(Capability.DrawParameters);
             }
 
-            context.AddExtension("SPV_KHR_shader_ballot");
-            context.AddExtension("SPV_KHR_subgroup_vote");
-
             Declarations.DeclareAll(context, info);
 
             if ((info.HelperFunctionsMask & NeedsInvocationIdMask) != 0)

Ryujinx.Graphics.Vulkan/Ryujinx.Graphics.Vulkan.csproj

@@ -13,7 +13,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="OpenTK.Windowing.GraphicsLibraryFramework" Version="4.7.2" />
+    <PackageReference Include="OpenTK.Windowing.GraphicsLibraryFramework" Version="4.7.5" />
     <PackageReference Include="shaderc.net" Version="0.1.0" />
     <PackageReference Include="Silk.NET.Vulkan" Version="2.10.1" />
     <PackageReference Include="Silk.NET.Vulkan.Extensions.EXT" Version="2.10.1" />

Ryujinx.Graphics.Vulkan/SyncManager.cs

@@ -11,6 +11,7 @@ namespace Ryujinx.Graphics.Vulkan
         {
             public ulong ID;
             public MultiFenceHolder Waitable;
+            public bool Signalled;
         }
 
         private ulong _firstHandle = 0;
@@ -45,6 +46,37 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
+        public ulong GetCurrent()
+        {
+            lock (_handles)
+            {
+                ulong lastHandle = _firstHandle;
+
+                foreach (SyncHandle handle in _handles)
+                {
+                    lock (handle)
+                    {
+                        if (handle.Waitable == null)
+                        {
+                            continue;
+                        }
+
+                        if (handle.ID > lastHandle)
+                        {
+                            bool signaled = handle.Signalled || handle.Waitable.WaitForFences(_gd.Api, _device, 0);
+                            if (signaled)
+                            {
+                                lastHandle = handle.ID;
+                                handle.Signalled = true;
+                            }
+                        }
+                    }
+                }
+
+                return lastHandle;
+            }
+        }
+
         public void Wait(ulong id)
         {
             SyncHandle result = null;
@@ -75,11 +107,15 @@ namespace Ryujinx.Graphics.Vulkan
                         return;
                     }
 
-                    bool signaled = result.Waitable.WaitForFences(_gd.Api, _device, 1000000000);
+                    bool signaled = result.Signalled || result.Waitable.WaitForFences(_gd.Api, _device, 1000000000);
                     if (!signaled)
                     {
                         Logger.Error?.PrintMsg(LogClass.Gpu, $"VK Sync Object {result.ID} failed to signal within 1000ms. Continuing...");
                     }
+                    else
+                    {
+                        result.Signalled = true;
+                    }
                 }
             }
         }

Ryujinx.Graphics.Vulkan/VulkanInitialization.cs

@@ -37,7 +37,6 @@ namespace Ryujinx.Graphics.Vulkan
         public static string[] RequiredExtensions { get; } = new string[]
         {
             KhrSwapchain.ExtensionName,
-            "VK_EXT_shader_subgroup_vote",
             ExtTransformFeedback.ExtensionName
         };
 

Ryujinx.Graphics.Vulkan/VulkanRenderer.cs

@@ -565,6 +565,11 @@ namespace Ryujinx.Graphics.Vulkan
             _syncManager.Wait(id);
         }
 
+        public ulong GetCurrentSync()
+        {
+            return _syncManager.GetCurrent();
+        }
+
         public void Screenshot()
         {
             _window.ScreenCaptureRequested = true;

Ryujinx.Headless.SDL2/Ryujinx.Headless.SDL2.csproj

@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="OpenTK.Core" Version="4.7.2" />
+    <PackageReference Include="OpenTK.Core" Version="4.7.5" />
     <PackageReference Include="Ryujinx.Graphics.Nvdec.Dependencies" Version="5.0.1-build10" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
   </ItemGroup>
 

Ryujinx.Memory/Tracking/ConcurrentBitmap.cs

@@ -41,7 +41,7 @@ namespace Ryujinx.Memory.Tracking
         {
             for (int i = 0; i < Masks.Length; i++)
             {
-                if (Volatile.Read(ref Masks[i]) != 0)
+                if (Interlocked.Read(ref Masks[i]) != 0)
                 {
                     return true;
                 }
@@ -62,7 +62,7 @@ namespace Ryujinx.Memory.Tracking
 
             long wordMask = 1L << wordBit;
 
-            return (Volatile.Read(ref Masks[wordIndex]) & wordMask) != 0;
+            return (Interlocked.Read(ref Masks[wordIndex]) & wordMask) != 0;
         }
 
         /// <summary>
@@ -86,7 +86,7 @@ namespace Ryujinx.Memory.Tracking
             int endBit = end & IntMask;
             long endMask = (long)(ulong.MaxValue >> (IntMask - endBit));
 
-            long startValue = Volatile.Read(ref Masks[startIndex]);
+            long startValue = Interlocked.Read(ref Masks[startIndex]);
 
             if (startIndex == endIndex)
             {
@@ -100,13 +100,13 @@ namespace Ryujinx.Memory.Tracking
 
             for (int i = startIndex + 1; i < endIndex; i++)
             {
-                if (Volatile.Read(ref Masks[i]) != 0)
+                if (Interlocked.Read(ref Masks[i]) != 0)
                 {
                     return true;
                 }
             }
 
-            long endValue = Volatile.Read(ref Masks[endIndex]);
+            long endValue = Interlocked.Read(ref Masks[endIndex]);
 
             if ((endValue & endMask) != 0)
             {
@@ -128,23 +128,14 @@ namespace Ryujinx.Memory.Tracking
 
             long wordMask = 1L << wordBit;
 
-            long existing;
-            long newValue;
-
-            do
+            if (value)
             {
-                existing = Volatile.Read(ref Masks[wordIndex]);
-
-                if (value)
-                {
-                    newValue = existing | wordMask;
-                }
-                else
-                {
-                    newValue = existing & ~wordMask;
-                }
+                Interlocked.Or(ref Masks[wordIndex], wordMask);
+            }
+            else
+            {
+                Interlocked.And(ref Masks[wordIndex], ~wordMask);
             }
-            while (Interlocked.CompareExchange(ref Masks[wordIndex], newValue, existing) != existing);
         }
 
         /// <summary>
@@ -154,7 +145,7 @@ namespace Ryujinx.Memory.Tracking
         {
             for (int i = 0; i < Masks.Length; i++)
             {
-                Volatile.Write(ref Masks[i], 0);
+                Interlocked.Exchange(ref Masks[i], 0);
             }
         }
     }

Ryujinx.Ui.Common/Ryujinx.Ui.Common.csproj

@@ -41,7 +41,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="DiscordRichPresence" Version="1.0.175" />
+    <PackageReference Include="DiscordRichPresence" Version="1.1.3.18" />
   </ItemGroup>
 
   <ItemGroup>

Ryujinx/Ryujinx.csproj

@@ -21,10 +21,10 @@
   <ItemGroup>
     <PackageReference Include="GtkSharp" Version="3.22.25.128" />
     <PackageReference Include="GtkSharp.Dependencies" Version="1.1.1" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
-    <PackageReference Include="OpenTK.Core" Version="4.7.2" />
     <PackageReference Include="Ryujinx.Graphics.Nvdec.Dependencies" Version="5.0.1-build10" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
     <PackageReference Include="Ryujinx.Audio.OpenAL.Dependencies" Version="1.21.0.1" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
-    <PackageReference Include="OpenTK.Graphics" Version="4.7.2" />
+    <PackageReference Include="OpenTK.Core" Version="4.7.5" />
+    <PackageReference Include="OpenTK.Graphics" Version="4.7.5" />
     <PackageReference Include="SPB" Version="0.0.4-build28" />
     <PackageReference Include="SharpZipLib" Version="1.4.1" />
     <PackageReference Include="SixLabors.ImageSharp" Version="1.0.4" />

distribution/macos/Info.plist

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleDevelopmentRegion</key>
+    <string>English</string>
+    <key>CFBundleExecutable</key>
+    <string>Ryujinx</string>
+    <key>CFBundleGetInfoString</key>
+    <string>Ryujinx</string>
+    <key>CFBundleIconFile</key>
+    <string>Ryujinx.icns</string>
+    <key>CFBundleTypeExtensions</key>
+    <array>
+        <string>nca</string>
+        <string>nro</string>
+        <string>nso</string>
+        <string>nsp</string>
+        <string>xci</string>
+    </array>
+    <key>CFBundleIdentifier</key>
+    <string>org.ryujinx.Ryujinx</string>
+    <key>CFBundleInfoDictionaryVersion</key>
+    <string>6.0</string>
+    <key>CFBundleLongVersionString</key>
+    <string>%%RYUJINX_BUILD_VERSION%%-%%RYUJINX_BUILD_GIT_HASH%%"</string>
+    <key>CFBundleName</key>
+    <string>Ryujinx</string>
+    <key>CFBundlePackageType</key>
+    <string>APPL</string>
+    <key>CFBundleShortVersionString</key>
+    <string>1.1</string>
+    <key>CFBundleSignature</key>
+    <string>????</string>
+    <key>CFBundleVersion</key>
+    <string>1.1.0</string>
+    <key>NSHighResolutionCapable</key>
+    <true/>
+    <key>CSResourcesFileMapped</key>
+    <true/>
+    <key>NSHumanReadableCopyright</key>
+    <string>Copyright © 2018 - 2022 Ryujinx Team and Contributors.</string>
+    <key>LSMinimumSystemVersion</key>
+    <string>11.0</string>
+</dict>
+</plist>

distribution/macos/bundle_fix_up.py

@@ -0,0 +1,609 @@
+import argparse
+import hashlib
+import os
+from pathlib import Path
+import platform
+import shutil
+import struct
+import subprocess
+from typing import List, Optional, Tuple
+
+parser = argparse.ArgumentParser(description="Fixup for MacOS application bundle")
+parser.add_argument("input_directory", help="Input directory (Application path)")
+parser.add_argument("executable_sub_path", help="Main executable sub path")
+
+# Use Apple LLVM on Darwin, otherwise standard LLVM.
+if platform.system() == "Darwin":
+    OTOOL = "otool"
+    INSTALL_NAME_TOOL = "install_name_tool"
+else:
+    OTOOL = shutil.which("llvm-otool")
+    if OTOOL is None:
+        for llvm_ver in [15, 14, 13]:
+            otool_path = shutil.which(f"llvm-otool-{llvm_ver}")
+            if otool_path is not None:
+                OTOOL = otool_path
+                INSTALL_NAME_TOOL = shutil.which(f"llvm-install-name-tool-{llvm_ver}")
+                break
+    else:
+        INSTALL_NAME_TOOL = shutil.which("llvm-install-name-tool")
+
+
+args = parser.parse_args()
+
+
+def get_dylib_id(dylib_path: Path) -> str:
+    res = subprocess.check_output([OTOOL, "-D", str(dylib_path.absolute())]).decode(
+        "utf-8"
+    )
+
+    return res.split("\n")[1]
+
+
+def get_dylib_dependencies(dylib_path: Path) -> List[str]:
+    output = (
+        subprocess.check_output([OTOOL, "-L", str(dylib_path.absolute())])
+        .decode("utf-8")
+        .split("\n")[1:]
+    )
+
+    res = []
+
+    for line in output:
+        line = line.strip()
+        index = line.find(" (compatibility version ")
+        if index == -1:
+            continue
+
+        line = line[:index]
+
+        res.append(line)
+
+    return res
+
+
+def replace_dylib_id(dylib_path: Path, new_id: str):
+    subprocess.check_call(
+        [INSTALL_NAME_TOOL, "-id", new_id, str(dylib_path.absolute())]
+    )
+
+
+def change_dylib_link(dylib_path: Path, old: str, new: str):
+    subprocess.check_call(
+        [INSTALL_NAME_TOOL, "-change", old, new, str(dylib_path.absolute())]
+    )
+
+
+def add_dylib_rpath(dylib_path: Path, rpath: str):
+    subprocess.check_call(
+        [INSTALL_NAME_TOOL, "-add_rpath", rpath, str(dylib_path.absolute())]
+    )
+
+
+def fixup_dylib(
+    dylib_path: Path,
+    replacement_path: str,
+    search_path: List[str],
+    content_directory: Path,
+):
+    dylib_id = get_dylib_id(dylib_path)
+    new_dylib_id = replacement_path + "/" + os.path.basename(dylib_id)
+    replace_dylib_id(dylib_path, new_dylib_id)
+
+    dylib_dependencies = get_dylib_dependencies(dylib_path)
+    dylib_new_mapping = {}
+
+    for dylib_dependency in dylib_dependencies:
+        if (
+            not dylib_dependency.startswith("@executable_path")
+            and not dylib_dependency.startswith("/usr/lib")
+            and not dylib_dependency.startswith("/System/Library")
+        ):
+            dylib_dependency_name = os.path.basename(dylib_dependency)
+            library_found = False
+            for library_base_path in search_path:
+                lib_path = Path(os.path.join(library_base_path, dylib_dependency_name))
+
+                if lib_path.exists():
+                    target_replacement_path = get_path_related_to_target_exec(
+                        content_directory, lib_path
+                    )
+
+                    dylib_new_mapping[dylib_dependency] = (
+                        target_replacement_path
+                        + "/"
+                        + os.path.basename(dylib_dependency)
+                    )
+                    library_found = True
+
+            if not library_found:
+                raise Exception(
+                    f"{dylib_id}: Cannot find dependency {dylib_dependency_name} for fixup"
+                )
+
+    for key in dylib_new_mapping:
+        change_dylib_link(dylib_path, key, dylib_new_mapping[key])
+
+
+FILE_TYPE_ASSEMBLY = 1
+
+ALIGN_REQUIREMENTS = 4096
+
+
+def parse_embedded_string(data: bytes) -> Tuple[bytes, str]:
+    first_byte = data[0]
+
+    if (first_byte & 0x80) == 0:
+        size = first_byte
+        data = data[1:]
+    else:
+        second_byte = data[1]
+
+        assert (second_byte & 0x80) == 0
+
+        size = (second_byte << 7) | (first_byte & 0x7F)
+
+        data = data[2:]
+
+    res = data[:size].decode("utf-8")
+    data = data[size:]
+
+    return (data, res)
+
+
+def write_embedded_string(file, string: str):
+    raw_str = string.encode("utf-8")
+    raw_str_len = len(raw_str)
+
+    assert raw_str_len < 0x7FFF
+
+    if raw_str_len > 0x7F:
+        file.write(struct.pack("b", raw_str_len & 0x7F | 0x80))
+        file.write(struct.pack("b", raw_str_len >> 7))
+    else:
+        file.write(struct.pack("b", raw_str_len))
+
+    file.write(raw_str)
+
+
+class BundleFileEntry(object):
+    offset: int
+    size: int
+    compressed_size: int
+    file_type: int
+    relative_path: str
+    data: bytes
+
+    def __init__(
+        self,
+        offset: int,
+        size: int,
+        compressed_size: int,
+        file_type: int,
+        relative_path: str,
+        data: bytes,
+    ) -> None:
+        self.offset = offset
+        self.size = size
+        self.compressed_size = compressed_size
+        self.file_type = file_type
+        self.relative_path = relative_path
+        self.data = data
+
+    def write(self, file):
+        self.offset = file.tell()
+
+        if (
+            self.file_type == FILE_TYPE_ASSEMBLY
+            and (self.offset % ALIGN_REQUIREMENTS) != 0
+        ):
+            padding_size = ALIGN_REQUIREMENTS - (self.offset % ALIGN_REQUIREMENTS)
+            file.write(b"\0" * padding_size)
+            self.offset += padding_size
+
+        file.write(self.data)
+
+    def write_header(self, file):
+        file.write(
+            struct.pack(
+                "QQQb", self.offset, self.size, self.compressed_size, self.file_type
+            )
+        )
+        write_embedded_string(file, self.relative_path)
+
+
+class BundleManifest(object):
+    major: int
+    minor: int
+    bundle_id: str
+    deps_json: BundleFileEntry
+    runtimeconfig_json: BundleFileEntry
+    flags: int
+    files: List[BundleFileEntry]
+
+    def __init__(
+        self,
+        major: int,
+        minor: int,
+        bundle_id: str,
+        deps_json: BundleFileEntry,
+        runtimeconfig_json: BundleFileEntry,
+        flags: int,
+        files: List[BundleFileEntry],
+    ) -> None:
+        self.major = major
+        self.minor = minor
+        self.bundle_id = bundle_id
+        self.deps_json = deps_json
+        self.runtimeconfig_json = runtimeconfig_json
+        self.flags = flags
+        self.files = files
+
+    def write(self, file) -> int:
+        for bundle_file in self.files:
+            bundle_file.write(file)
+
+        bundle_header_offset = file.tell()
+        file.write(struct.pack("iiI", self.major, self.minor, len(self.files)))
+        write_embedded_string(file, self.bundle_id)
+
+        if self.deps_json is not None:
+            deps_json_location_offset = self.deps_json.offset
+            deps_json_location_size = self.deps_json.size
+        else:
+            deps_json_location_offset = 0
+            deps_json_location_size = 0
+
+        if self.runtimeconfig_json is not None:
+            runtimeconfig_json_location_offset = self.runtimeconfig_json.offset
+            runtimeconfig_json_location_size = self.runtimeconfig_json.size
+        else:
+            runtimeconfig_json_location_offset = 0
+            runtimeconfig_json_location_size = 0
+
+        file.write(
+            struct.pack("qq", deps_json_location_offset, deps_json_location_size)
+        )
+        file.write(
+            struct.pack(
+                "qq",
+                runtimeconfig_json_location_offset,
+                runtimeconfig_json_location_size,
+            )
+        )
+        file.write(struct.pack("q", self.flags))
+
+        for bundle_file in self.files:
+            bundle_file.write_header(file)
+
+        return bundle_header_offset
+
+
+def read_file_entry(
+    raw_data: bytes, header_bytes: bytes
+) -> Tuple[bytes, BundleFileEntry]:
+    (
+        offset,
+        size,
+        compressed_size,
+        file_type,
+    ) = struct.unpack("QQQb", header_bytes[:0x19])
+    (header_bytes, relative_path) = parse_embedded_string(header_bytes[0x19:])
+
+    target_size = compressed_size
+
+    if target_size == 0:
+        target_size = size
+
+    return (
+        header_bytes,
+        BundleFileEntry(
+            offset,
+            size,
+            compressed_size,
+            file_type,
+            relative_path,
+            raw_data[offset : offset + target_size],
+        ),
+    )
+
+
+def get_dotnet_bundle_data(data: bytes) -> Optional[Tuple[int, int, BundleManifest]]:
+    offset = data.find(hashlib.sha256(b".net core bundle\n").digest())
+
+    if offset == -1:
+        return None
+
+    raw_header_offset = data[offset - 8 : offset]
+    (header_offset,) = struct.unpack("q", raw_header_offset)
+    header_bytes = data[header_offset:]
+
+    (
+        major,
+        minor,
+        files_count,
+    ) = struct.unpack("iiI", header_bytes[:0xC])
+    header_bytes = header_bytes[0xC:]
+
+    (header_bytes, bundle_id) = parse_embedded_string(header_bytes)
+
+    # v2 header
+    (
+        deps_json_location_offset,
+        deps_json_location_size,
+    ) = struct.unpack("qq", header_bytes[:0x10])
+    (
+        runtimeconfig_json_location_offset,
+        runtimeconfig_json_location_size,
+    ) = struct.unpack("qq", header_bytes[0x10:0x20])
+    (flags,) = struct.unpack("q", header_bytes[0x20:0x28])
+    header_bytes = header_bytes[0x28:]
+
+    files = []
+
+    deps_json = None
+    runtimeconfig_json = None
+
+    for _ in range(files_count):
+        (header_bytes, file_entry) = read_file_entry(data, header_bytes)
+
+        files.append(file_entry)
+
+        if file_entry.offset == deps_json_location_offset:
+            deps_json = file_entry
+        elif file_entry.offset == runtimeconfig_json_location_offset:
+            runtimeconfig_json = file_entry
+
+    file_entry = files[0]
+
+    return (
+        file_entry.offset,
+        header_offset,
+        BundleManifest(
+            major, minor, bundle_id, deps_json, runtimeconfig_json, flags, files
+        ),
+    )
+
+
+LC_SYMTAB = 0x2
+LC_SEGMENT_64 = 0x19
+LC_CODE_SIGNATURE = 0x1D
+
+
+def fixup_linkedit(file, data: bytes, new_size: int):
+    offset = 0
+
+    (
+        macho_magic,
+        macho_cputype,
+        macho_cpusubtype,
+        macho_filetype,
+        macho_ncmds,
+        macho_sizeofcmds,
+        macho_flags,
+        macho_reserved,
+    ) = struct.unpack("IiiIIIII", data[offset : offset + 0x20])
+
+    offset += 0x20
+
+    linkedit_offset = None
+    symtab_offset = None
+    codesign_offset = None
+
+    for _ in range(macho_ncmds):
+        (cmd, cmdsize) = struct.unpack("II", data[offset : offset + 8])
+
+        if cmd == LC_SEGMENT_64:
+            (
+                cmd,
+                cmdsize,
+                segname_raw,
+                vmaddr,
+                vmsize,
+                fileoff,
+                filesize,
+                maxprot,
+                initprot,
+                nsects,
+                flags,
+            ) = struct.unpack("II16sQQQQiiII", data[offset : offset + 72])
+            segname = segname_raw.decode("utf-8").split("\0")[0]
+
+            if segname == "__LINKEDIT":
+                linkedit_offset = offset
+        elif cmd == LC_SYMTAB:
+            symtab_offset = offset
+        elif cmd == LC_CODE_SIGNATURE:
+            codesign_offset = offset
+
+        offset += cmdsize
+        pass
+
+    assert linkedit_offset is not None and symtab_offset is not None
+
+    # If there is a codesign section, clean it up.
+    if codesign_offset is not None:
+        (
+            codesign_cmd,
+            codesign_cmdsize,
+            codesign_dataoff,
+            codesign_datasize,
+        ) = struct.unpack("IIII", data[codesign_offset : codesign_offset + 16])
+        file.seek(codesign_offset)
+        file.write(b"\0" * codesign_cmdsize)
+
+        macho_ncmds -= 1
+        macho_sizeofcmds -= codesign_cmdsize
+        file.seek(0)
+        file.write(
+            struct.pack(
+                "IiiIIIII",
+                macho_magic,
+                macho_cputype,
+                macho_cpusubtype,
+                macho_filetype,
+                macho_ncmds,
+                macho_sizeofcmds,
+                macho_flags,
+                macho_reserved,
+            )
+        )
+
+        file.seek(codesign_dataoff)
+        file.write(b"\0" * codesign_datasize)
+
+    (
+        symtab_cmd,
+        symtab_cmdsize,
+        symtab_symoff,
+        symtab_nsyms,
+        symtab_stroff,
+        symtab_strsize,
+    ) = struct.unpack("IIIIII", data[symtab_offset : symtab_offset + 24])
+
+    symtab_strsize = new_size - symtab_stroff
+
+    new_symtab = struct.pack(
+        "IIIIII",
+        symtab_cmd,
+        symtab_cmdsize,
+        symtab_symoff,
+        symtab_nsyms,
+        symtab_stroff,
+        symtab_strsize,
+    )
+
+    file.seek(symtab_offset)
+    file.write(new_symtab)
+
+    (
+        linkedit_cmd,
+        linkedit_cmdsize,
+        linkedit_segname_raw,
+        linkedit_vmaddr,
+        linkedit_vmsize,
+        linkedit_fileoff,
+        linkedit_filesize,
+        linkedit_maxprot,
+        linkedit_initprot,
+        linkedit_nsects,
+        linkedit_flags,
+    ) = struct.unpack("II16sQQQQiiII", data[linkedit_offset : linkedit_offset + 72])
+
+    linkedit_filesize = new_size - linkedit_fileoff
+    linkedit_vmsize = linkedit_filesize
+
+    new_linkedit = struct.pack(
+        "II16sQQQQiiII",
+        linkedit_cmd,
+        linkedit_cmdsize,
+        linkedit_segname_raw,
+        linkedit_vmaddr,
+        linkedit_vmsize,
+        linkedit_fileoff,
+        linkedit_filesize,
+        linkedit_maxprot,
+        linkedit_initprot,
+        linkedit_nsects,
+        linkedit_flags,
+    )
+    file.seek(linkedit_offset)
+    file.write(new_linkedit)
+
+
+def write_bundle_data(
+    output,
+    old_bundle_base_offset: int,
+    new_bundle_base_offset: int,
+    bundle: BundleManifest,
+) -> int:
+    # Write bundle data
+    bundle_header_offset = bundle.write(output)
+    total_size = output.tell()
+
+    # Patch the header position
+    offset = file_data.find(hashlib.sha256(b".net core bundle\n").digest())
+    output.seek(offset - 8)
+    output.write(struct.pack("q", bundle_header_offset))
+
+    return total_size - new_bundle_base_offset
+
+
+input_directory: Path = Path(args.input_directory)
+content_directory: Path = Path(os.path.join(args.input_directory, "Contents"))
+executable_path: Path = Path(os.path.join(content_directory, args.executable_sub_path))
+
+
+def get_path_related_to_other_path(a: Path, b: Path) -> str:
+    temp = b
+
+    parts = []
+
+    while temp != a:
+        temp = temp.parent
+        parts.append(temp.name)
+
+    parts.remove(parts[-1])
+    parts.reverse()
+
+    return "/".join(parts)
+
+
+def get_path_related_to_target_exec(input_directory: Path, path: Path):
+    return "@executable_path/../" + get_path_related_to_other_path(
+        input_directory, path
+    )
+
+
+search_path = [
+    Path(os.path.join(content_directory, "Frameworks")),
+    Path(os.path.join(content_directory, "Resources/lib")),
+]
+
+
+for path in content_directory.rglob("**/*.dylib"):
+    current_search_path = [path.parent]
+    current_search_path.extend(search_path)
+
+    fixup_dylib(
+        path,
+        get_path_related_to_target_exec(content_directory, path),
+        current_search_path,
+        content_directory,
+    )
+
+for path in content_directory.rglob("**/*.so"):
+    current_search_path = [path.parent]
+    current_search_path.extend(search_path)
+
+    fixup_dylib(
+        path,
+        get_path_related_to_target_exec(content_directory, path),
+        current_search_path,
+        content_directory,
+    )
+
+
+with open(executable_path, "rb") as input:
+    file_data = input.read()
+
+
+(bundle_base_offset, bundle_header_offset, bundle) = get_dotnet_bundle_data(file_data)
+
+add_dylib_rpath(executable_path, "@executable_path/../Frameworks/")
+
+# Recent "vanilla" version of LLVM (LLVM 13 and upper) seems to really dislike how .NET package its assemblies.
+# As a result, after execution of install_name_tool it will have "fixed" the symtab resulting in a missing .NET bundle...
+# To mitigate that, we check if the bundle offset inside the binary is valid after install_name_tool and readd .NET bundle if not.
+output_file_size = os.stat(executable_path).st_size
+if output_file_size < bundle_header_offset:
+    print("LLVM broke the .NET bundle, readding bundle data...")
+    with open(executable_path, "r+b") as output:
+        file_data = output.read()
+        bundle_data_size = write_bundle_data(
+            output, bundle_base_offset, output_file_size, bundle
+        )
+
+        # Now patch the __LINKEDIT section
+        new_size = output_file_size + bundle_data_size
+        fixup_linkedit(output, file_data, new_size)

distribution/macos/construct_universal_dylib.py

@@ -0,0 +1,95 @@
+import argparse
+import os
+from pathlib import Path
+import platform
+import shutil
+import subprocess
+
+parser = argparse.ArgumentParser(
+    description="Construct Universal dylibs for nuget package"
+)
+parser.add_argument(
+    "arm64_input_directory", help="ARM64 Input directory containing dylibs"
+)
+parser.add_argument(
+    "x86_64_input_directory", help="x86_64 Input directory containing dylibs"
+)
+parser.add_argument("output_directory", help="Output directory")
+parser.add_argument("rglob", help="rglob")
+
+args = parser.parse_args()
+
+# Use Apple LLVM on Darwin, otherwise standard LLVM.
+if platform.system() == "Darwin":
+    LIPO = "lipo"
+else:
+    LIPO = shutil.which("llvm-lipo")
+
+    if LIPO is None:
+        for llvm_ver in [15, 14, 13]:
+            lipo_path = shutil.which(f"llvm-lipo-{llvm_ver}")
+            if lipo_path is not None:
+                LIPO = lipo_path
+                break
+
+if LIPO is None:
+    raise Exception("Cannot find a valid location for LLVM lipo!")
+
+arm64_input_directory: Path = Path(args.arm64_input_directory)
+x86_64_input_directory: Path = Path(args.x86_64_input_directory)
+output_directory: Path = Path(args.output_directory)
+rglob = args.rglob
+
+
+def get_new_name(
+    input_directory: Path, output_directory: str, input_dylib_path: Path
+) -> Path:
+    input_component = str(input_dylib_path).replace(str(input_directory), "")[1:]
+    return Path(os.path.join(output_directory, input_component))
+
+
+def is_fat_file(dylib_path: Path) -> str:
+    res = subprocess.check_output([LIPO, "-info", str(dylib_path.absolute())]).decode(
+        "utf-8"
+    )
+
+    return not res.split("\n")[0].startswith("Non-fat file")
+
+
+def construct_universal_dylib(
+    arm64_input_dylib_path: Path, x86_64_input_dylib_path: Path, output_dylib_path: Path
+):
+    if output_dylib_path.exists() or output_dylib_path.is_symlink():
+        os.remove(output_dylib_path)
+
+    os.makedirs(output_dylib_path.parent, exist_ok=True)
+
+    if arm64_input_dylib_path.is_symlink():
+        os.symlink(
+            os.path.basename(arm64_input_dylib_path.resolve()), output_dylib_path
+        )
+    else:
+        if is_fat_file(arm64_input_dylib_path) or not x86_64_input_dylib_path.exists():
+            with open(output_dylib_path, "wb") as dst:
+                with open(arm64_input_dylib_path, "rb") as src:
+                    dst.write(src.read())
+        else:
+            subprocess.check_call(
+                [
+                    LIPO,
+                    str(arm64_input_dylib_path.absolute()),
+                    str(x86_64_input_dylib_path.absolute()),
+                    "-output",
+                    str(output_dylib_path.absolute()),
+                    "-create",
+                ]
+            )
+
+
+print(rglob)
+for path in arm64_input_directory.rglob("**/*.dylib"):
+    construct_universal_dylib(
+        path,
+        get_new_name(arm64_input_directory, x86_64_input_directory, path),
+        get_new_name(arm64_input_directory, output_directory, path),
+    )

distribution/macos/create_app_bundle.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -e
+
+PUBLISH_DIRECTORY=$1
+OUTPUT_DIRECTORY=$2
+ENTITLEMENTS_FILE_PATH=$3
+
+APP_BUNDLE_DIRECTORY=$OUTPUT_DIRECTORY/Ryujinx.app
+
+rm -rf $APP_BUNDLE_DIRECTORY
+mkdir -p $APP_BUNDLE_DIRECTORY/Contents
+mkdir $APP_BUNDLE_DIRECTORY/Contents/Frameworks
+mkdir $APP_BUNDLE_DIRECTORY/Contents/MacOS
+mkdir $APP_BUNDLE_DIRECTORY/Contents/Resources
+
+# Copy executables first
+cp $PUBLISH_DIRECTORY/Ryujinx.Ava $APP_BUNDLE_DIRECTORY/Contents/MacOS/Ryujinx
+chmod u+x $APP_BUNDLE_DIRECTORY/Contents/MacOS/Ryujinx
+
+# Then all libraries
+cp $PUBLISH_DIRECTORY/*.dylib $APP_BUNDLE_DIRECTORY/Contents/Frameworks
+
+# Then resources
+cp Info.plist $APP_BUNDLE_DIRECTORY/Contents
+cp Ryujinx.icns $APP_BUNDLE_DIRECTORY/Contents/Resources/Ryujinx.icns
+cp -r $PUBLISH_DIRECTORY/THIRDPARTY.md $APP_BUNDLE_DIRECTORY/Contents/Resources
+
+echo -n "APPL????" > $APP_BUNDLE_DIRECTORY/Contents/PkgInfo
+
+# Fixup libraries and executable
+python3 bundle_fix_up.py $APP_BUNDLE_DIRECTORY MacOS/Ryujinx
+
+# Now sign it
+if ! [ -x "$(command -v codesign)" ];
+then
+    if ! [ -x "$(command -v rcodesign)" ];
+    then
+        echo "Cannot find rcodesign on your system, please install rcodesign."
+        exit 1
+    fi
+
+    # NOTE: Currently require https://github.com/indygreg/apple-platform-rs/pull/44 to work on other OSes.
+    # cargo install --git "https://github.com/marysaka/apple-platform-rs" --branch "fix/adhoc-app-bundle" apple-codesign --bin "rcodesign"
+    echo "Usign rcodesign for ad-hoc signing"
+    rcodesign sign --entitlements-xml-path $ENTITLEMENTS_FILE_PATH $APP_BUNDLE_DIRECTORY
+else
+    echo "Usign codesign for ad-hoc signing"
+    codesign --entitlements $ENTITLEMENTS_FILE_PATH -f --deep -s - $APP_BUNDLE_DIRECTORY
+fi
+

distribution/macos/create_macos_release.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -e
+
+if [ "$#" -ne 6 ]; then
+    echo "usage <BASE_DIR> <TEMP_DIRECTORY> <OUTPUT_DIRECTORY> <ENTITLEMENTS_FILE_PATH> <VERSION> <SOURCE_REVISION_ID>"
+    exit 1
+fi
+
+mkdir -p $1
+mkdir -p $2
+mkdir -p $3
+
+BASE_DIR=$(readlink -f $1)
+TEMP_DIRECTORY=$(readlink -f $2)
+OUTPUT_DIRECTORY=$(readlink -f $3)
+ENTITLEMENTS_FILE_PATH=$(readlink -f $4)
+VERSION=$5
+SOURCE_REVISION_ID=$6
+
+RELEASE_TAR_FILE_NAME=Ryujinx-$VERSION-macos_universal.app.tar
+ARM64_APP_BUNDLE=$TEMP_DIRECTORY/output_arm64/Ryujinx.app
+X64_APP_BUNDLE=$TEMP_DIRECTORY/output_x64/Ryujinx.app
+UNIVERSAL_APP_BUNDLE=$OUTPUT_DIRECTORY/Ryujinx.app
+EXECUTABLE_SUB_PATH=Contents/MacOS/Ryujinx
+
+rm -rf $TEMP_DIRECTORY
+mkdir -p $TEMP_DIRECTORY
+
+DOTNET_COMMON_ARGS="-p:DebugType=embedded -p:Version=$VERSION -p:SourceRevisionId=$SOURCE_REVISION_ID -p:ExtraDefineConstants=DISABLE_UPDATER --self-contained true"
+
+dotnet restore
+dotnet build -c Release Ryujinx.Ava
+dotnet publish -c Release -r osx-arm64 -o $TEMP_DIRECTORY/publish_arm64 $DOTNET_COMMON_ARGS Ryujinx.Ava
+dotnet publish -c Release -r osx-x64 -o $TEMP_DIRECTORY/publish_x64 $DOTNET_COMMON_ARGS Ryujinx.Ava
+
+# Get ride of the support library for ARMeilleur for x64 (that's only for arm64)
+rm -rf $TEMP_DIRECTORY/publish_x64/libarmeilleure-jitsupport.dylib
+
+# Get ride of libsoundio from arm64 builds as we don't have a arm64 variant
+# TODO: remove this once done
+rm -rf $TEMP_DIRECTORY/publish_arm64/libsoundio.dylib
+
+pushd $BASE_DIR/distribution/macos
+./create_app_bundle.sh $TEMP_DIRECTORY/publish_x64 $TEMP_DIRECTORY/output_x64 $ENTITLEMENTS_FILE_PATH
+./create_app_bundle.sh $TEMP_DIRECTORY/publish_arm64 $TEMP_DIRECTORY/output_arm64 $ENTITLEMENTS_FILE_PATH
+popd
+
+rm -rf $UNIVERSAL_APP_BUNDLE
+mkdir -p $OUTPUT_DIRECTORY
+
+# Let's copy one of the two different app bundle and remove the executable
+cp -R $ARM64_APP_BUNDLE $UNIVERSAL_APP_BUNDLE
+rm $UNIVERSAL_APP_BUNDLE/$EXECUTABLE_SUB_PATH
+
+# Make it libraries universal
+python3 $BASE_DIR/distribution/macos/construct_universal_dylib.py $ARM64_APP_BUNDLE $X64_APP_BUNDLE $UNIVERSAL_APP_BUNDLE "**/*.dylib"
+
+if ! [ -x "$(command -v lipo)" ];
+then
+    if ! [ -x "$(command -v llvm-lipo-14)" ];
+    then
+        LIPO=llvm-lipo
+    else
+        LIPO=llvm-lipo-14
+    fi
+else
+    LIPO=lipo
+fi
+
+# Make it the executable universal
+$LIPO $ARM64_APP_BUNDLE/$EXECUTABLE_SUB_PATH $X64_APP_BUNDLE/$EXECUTABLE_SUB_PATH -output $UNIVERSAL_APP_BUNDLE/$EXECUTABLE_SUB_PATH -create
+
+# Patch up the Info.plist to have appropriate version
+sed -r -i.bck "s/\%\%RYUJINX_BUILD_VERSION\%\%/$VERSION/g;" $UNIVERSAL_APP_BUNDLE/Contents/Info.plist
+sed -r -i.bck "s/\%\%RYUJINX_BUILD_GIT_HASH\%\%/$SOURCE_REVISION_ID/g;" $UNIVERSAL_APP_BUNDLE/Contents/Info.plist
+rm $UNIVERSAL_APP_BUNDLE/Contents/Info.plist.bck
+
+# Now sign it
+if ! [ -x "$(command -v codesign)" ];
+then
+    if ! [ -x "$(command -v rcodesign)" ];
+    then
+        echo "Cannot find rcodesign on your system, please install rcodesign."
+        exit 1
+    fi
+
+    # NOTE: Currently require https://github.com/indygreg/apple-platform-rs/pull/44 to work on other OSes.
+    # cargo install --git "https://github.com/marysaka/apple-platform-rs" --branch "fix/adhoc-app-bundle" apple-codesign --bin "rcodesign"
+    echo "Usign rcodesign for ad-hoc signing"
+    rcodesign sign --entitlements-xml-path $ENTITLEMENTS_FILE_PATH $UNIVERSAL_APP_BUNDLE
+else
+    echo "Usign codesign for ad-hoc signing"
+    codesign --entitlements $ENTITLEMENTS_FILE_PATH -f --deep -s - $UNIVERSAL_APP_BUNDLE
+fi
+
+echo "Creating archive"
+pushd $OUTPUT_DIRECTORY
+tar --exclude "Ryujinx.app/Contents/MacOS/Ryujinx" -cvf $RELEASE_TAR_FILE_NAME Ryujinx.app 1> /dev/null
+python3 $BASE_DIR/distribution/misc/add_tar_exec.py $RELEASE_TAR_FILE_NAME "Ryujinx.app/Contents/MacOS/Ryujinx" "Ryujinx.app/Contents/MacOS/Ryujinx"
+gzip -9 < $RELEASE_TAR_FILE_NAME > $RELEASE_TAR_FILE_NAME.gz
+rm $RELEASE_TAR_FILE_NAME
+popd
+
+echo "Done"

distribution/macos/entitlements.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
+"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+    <true/>
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
+    <key>com.apple.security.cs.disable-executable-page-protection</key>
+    <true/>
+    <key>com.apple.security.cs.debugger</key>
+    <true/>
+    <key>com.apple.security.get-task-allow</key>
+    <true/>
+    <key>com.apple.security.hypervisor</key>
+    <true/>
+</dict>
+</plist>

distribution/misc/add_tar_exec.py

@@ -0,0 +1,24 @@
+import argparse
+from io import BytesIO
+import tarfile
+
+parser = argparse.ArgumentParser(
+    description="Add the main binary to a tar and force it to be executable"
+)
+parser.add_argument("input_tar_file", help="input tar file")
+parser.add_argument("main_binary_path", help="Main executable path")
+parser.add_argument("main_binary_tar_path", help="Main executable tar path")
+
+args = parser.parse_args()
+input_tar_file = args.input_tar_file
+main_binary_path = args.main_binary_path
+main_binary_tar_path = args.main_binary_tar_path
+
+with open(main_binary_path, "rb") as f:
+    with tarfile.open(input_tar_file, "a") as tar:
+        data = f.read()
+        tar_info = tarfile.TarInfo(main_binary_tar_path)
+        tar_info.mode = 0o755
+        tar_info.size = len(data)
+
+        tar.addfile(tar_info, BytesIO(data))