Use pooled memory and avoid memory copies (#6691)

* perf: use ByteMemoryPool

* feat: KPageTableBase/KPageTable new methods to read and write `ReadOnlySequence<byte>`

* new: add IWritableBlock.Write(ulong, ReadOnlySequence<byte>) with default impl

* perf: use GetReadOnlySequence() instead of GetSpan()

* perf: make `Parcel` IDisposable, use `ByteMemoryPool` for internal allocation, and make Parcel consumers dispose of it

* remove comment about copySize

* remove unnecessary Clear()

Directory.Packages.props

@@ -42,7 +42,7 @@
     <PackageVersion Include="Silk.NET.Vulkan" Version="2.16.0" />
     <PackageVersion Include="Silk.NET.Vulkan.Extensions.EXT" Version="2.16.0" />
     <PackageVersion Include="Silk.NET.Vulkan.Extensions.KHR" Version="2.16.0" />
-    <PackageVersion Include="SixLabors.ImageSharp" Version="2.1.7" />
+    <PackageVersion Include="SixLabors.ImageSharp" Version="2.1.8" />
     <PackageVersion Include="SixLabors.ImageSharp.Drawing" Version="1.0.0" />
     <PackageVersion Include="SPB" Version="0.0.4-build32" />
     <PackageVersion Include="System.IO.Hashing" Version="8.0.0" />

src/Ryujinx.Audio.Backends.SDL2/SDL2HardwareDeviceSession.cs

@@ -1,8 +1,10 @@
 using Ryujinx.Audio.Backends.Common;
 using Ryujinx.Audio.Common;
 using Ryujinx.Common.Logging;
+using Ryujinx.Common.Memory;
 using Ryujinx.Memory;
 using System;
+using System.Buffers;
 using System.Collections.Concurrent;
 using System.Threading;
 
@@ -87,7 +89,9 @@ namespace Ryujinx.Audio.Backends.SDL2
                 return;
             }
 
-            byte[] samples = new byte[frameCount * _bytesPerFrame];
+            using IMemoryOwner<byte> samplesOwner = ByteMemoryPool.Rent(frameCount * _bytesPerFrame);
+
+            Span<byte> samples = samplesOwner.Memory.Span;
 
             _ringBuffer.Read(samples, 0, samples.Length);
 

src/Ryujinx.Audio.Backends.SoundIo/SoundIoHardwareDeviceSession.cs

@@ -1,8 +1,10 @@
 using Ryujinx.Audio.Backends.Common;
 using Ryujinx.Audio.Backends.SoundIo.Native;
 using Ryujinx.Audio.Common;
+using Ryujinx.Common.Memory;
 using Ryujinx.Memory;
 using System;
+using System.Buffers;
 using System.Collections.Concurrent;
 using System.Runtime.CompilerServices;
 using System.Threading;
@@ -37,7 +39,7 @@ namespace Ryujinx.Audio.Backends.SoundIo
             _outputStream = _driver.OpenStream(RequestedSampleFormat, RequestedSampleRate, RequestedChannelCount);
             _outputStream.WriteCallback += Update;
             _outputStream.Volume = requestedVolume;
-            // TODO: Setup other callbacks (errors, ect).
+            // TODO: Setup other callbacks (errors, etc.)
 
             _outputStream.Open();
         }
@@ -120,7 +122,9 @@ namespace Ryujinx.Audio.Backends.SoundIo
 
             int channelCount = areas.Length;
 
-            byte[] samples = new byte[frameCount * bytesPerFrame];
+            using IMemoryOwner<byte> samplesOwner = ByteMemoryPool.Rent(frameCount * bytesPerFrame);
+
+            Span<byte> samples = samplesOwner.Memory.Span;
 
             _ringBuffer.Read(samples, 0, samples.Length);
 

src/Ryujinx.Audio/Renderer/Server/BehaviourContext.cs

@@ -212,7 +212,7 @@ namespace Ryujinx.Audio.Renderer.Server
         /// <summary>
         /// Check if the audio renderer should fix the GC-ADPCM context not being provided to the DSP.
         /// </summary>
-        /// <returns>True if if the audio renderer should fix it.</returns>
+        /// <returns>True if the audio renderer should fix it.</returns>
         public bool IsAdpcmLoopContextBugFixed()
         {
             return CheckFeatureSupported(UserRevision, BaseRevisionMagic + Revision2);

src/Ryujinx.Common/Extensions/SequenceReaderExtensions.cs

@@ -44,7 +44,7 @@ namespace Ryujinx.Common.Extensions
         /// <remarks>
         /// DO NOT use <paramref name="copyDestinationIfRequiredDoNotUse"/> after calling this method, as it will only
         /// contain a value if the value couldn't be referenced directly because it spans multiple <see cref="ReadOnlyMemory{Byte}"/> segments.
-        /// To discourage use, it is recommended to to call this method like the following:
+        /// To discourage use, it is recommended to call this method like the following:
         /// <c>
         ///     ref readonly MyStruct value = ref sequenceReader.GetRefOrRefToCopy{MyStruct}(out _);
         /// </c>

src/Ryujinx.Graphics.Gpu/Engine/GPFifo/GPFifoClass.cs

@@ -157,6 +157,9 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo
             }
             else if (operation == SyncpointbOperation.Incr)
             {
+                // "Unbind" render targets since a syncpoint increment might indicate future CPU access for the textures.
+                _parent.TextureManager.RefreshModifiedTextures();
+
                 _context.CreateHostSyncIfNeeded(HostSyncFlags.StrictSyncpoint);
                 _context.Synchronization.IncrementSyncpoint(syncpointId);
             }

src/Ryujinx.Graphics.Gpu/Engine/GPFifo/GPFifoProcessor.cs

@@ -4,6 +4,7 @@ using Ryujinx.Graphics.Gpu.Engine.Dma;
 using Ryujinx.Graphics.Gpu.Engine.InlineToMemory;
 using Ryujinx.Graphics.Gpu.Engine.Threed;
 using Ryujinx.Graphics.Gpu.Engine.Twod;
+using Ryujinx.Graphics.Gpu.Image;
 using Ryujinx.Graphics.Gpu.Memory;
 using System;
 using System.Runtime.CompilerServices;
@@ -28,6 +29,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo
         /// </summary>
         public MemoryManager MemoryManager => _channel.MemoryManager;
 
+        /// <summary>
+        /// Channel texture manager.
+        /// </summary>
+        public TextureManager TextureManager => _channel.TextureManager;
+
         /// <summary>
         /// 3D Engine.
         /// </summary>

src/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs

@@ -770,7 +770,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 ? _channel.BufferManager.GetComputeUniformBufferAddress(textureBufferIndex)
                 : _channel.BufferManager.GetGraphicsUniformBufferAddress(stageIndex, textureBufferIndex);
 
-            int handle = textureBufferAddress != 0
+            int handle = textureBufferAddress != MemoryManager.PteUnmapped
                 ? _channel.MemoryManager.Physical.Read<int>(textureBufferAddress + (uint)textureWordOffset * 4)
                 : 0;
 
@@ -790,7 +790,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                         ? _channel.BufferManager.GetComputeUniformBufferAddress(samplerBufferIndex)
                         : _channel.BufferManager.GetGraphicsUniformBufferAddress(stageIndex, samplerBufferIndex);
 
-                    samplerHandle = samplerBufferAddress != 0
+                    samplerHandle = samplerBufferAddress != MemoryManager.PteUnmapped
                         ? _channel.MemoryManager.Physical.Read<int>(samplerBufferAddress + (uint)samplerWordOffset * 4)
                         : 0;
                 }

src/Ryujinx.Graphics.Vulkan/CommandBufferPool.cs

@@ -302,12 +302,12 @@ namespace Ryujinx.Graphics.Vulkan
                         SubmitInfo sInfo = new()
                         {
                             SType = StructureType.SubmitInfo,
-                            WaitSemaphoreCount = waitSemaphores != null ? (uint)waitSemaphores.Length : 0,
+                            WaitSemaphoreCount = !waitSemaphores.IsEmpty ? (uint)waitSemaphores.Length : 0,
                             PWaitSemaphores = pWaitSemaphores,
                             PWaitDstStageMask = pWaitDstStageMask,
                             CommandBufferCount = 1,
                             PCommandBuffers = &commandBuffer,
-                            SignalSemaphoreCount = signalSemaphores != null ? (uint)signalSemaphores.Length : 0,
+                            SignalSemaphoreCount = !signalSemaphores.IsEmpty ? (uint)signalSemaphores.Length : 0,
                             PSignalSemaphores = pSignalSemaphores,
                         };
 

src/Ryujinx.HLE/HOS/Kernel/Ipc/KServerSession.cs

@@ -570,7 +570,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Ipc
                 }
                 else
                 {
-                    serverProcess.CpuMemory.Write(copyDst, clientProcess.CpuMemory.GetSpan(copySrc, (int)copySize));
+                    serverProcess.CpuMemory.Write(copyDst, clientProcess.CpuMemory.GetReadOnlySequence(copySrc, (int)copySize));
                 }
 
                 if (clientResult != Result.Success)
@@ -858,7 +858,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Ipc
                 }
                 else
                 {
-                    clientProcess.CpuMemory.Write(copyDst, serverProcess.CpuMemory.GetSpan(copySrc, (int)copySize));
+                    clientProcess.CpuMemory.Write(copyDst, serverProcess.CpuMemory.GetReadOnlySequence(copySrc, (int)copySize));
                 }
             }
 

src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs

@@ -2,6 +2,7 @@ using Ryujinx.Horizon.Common;
 using Ryujinx.Memory;
 using Ryujinx.Memory.Range;
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Diagnostics;
 
@@ -34,6 +35,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             }
         }
 
+        /// <inheritdoc/>
+        protected override ReadOnlySequence<byte> GetReadOnlySequence(ulong va, int size)
+        {
+            return _cpuMemory.GetReadOnlySequence(va, size);
+        }
+
         /// <inheritdoc/>
         protected override ReadOnlySpan<byte> GetSpan(ulong va, int size)
         {
@@ -247,6 +254,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             _cpuMemory.SignalMemoryTracking(va, size, write);
         }
 
+        /// <inheritdoc/>
+        protected override void Write(ulong va, ReadOnlySequence<byte> data)
+        {
+            _cpuMemory.Write(va, data);
+        }
+
         /// <inheritdoc/>
         protected override void Write(ulong va, ReadOnlySpan<byte> data)
         {

src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs

@@ -5,6 +5,7 @@ using Ryujinx.Horizon.Common;
 using Ryujinx.Memory;
 using Ryujinx.Memory.Range;
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Diagnostics;
 
@@ -1568,7 +1569,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
                     while (size > 0)
                     {
-                        ulong copySize = 0x100000; // Copy chunck size. Any value will do, moderate sizes are recommended.
+                        ulong copySize = int.MaxValue;
 
                         if (copySize > size)
                         {
@@ -1577,11 +1578,11 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
                         if (toServer)
                         {
-                            currentProcess.CpuMemory.Write(serverAddress, GetSpan(clientAddress, (int)copySize));
+                            currentProcess.CpuMemory.Write(serverAddress, GetReadOnlySequence(clientAddress, (int)copySize));
                         }
                         else
                         {
-                            Write(clientAddress, currentProcess.CpuMemory.GetSpan(serverAddress, (int)copySize));
+                            Write(clientAddress, currentProcess.CpuMemory.GetReadOnlySequence(serverAddress, (int)copySize));
                         }
 
                         serverAddress += copySize;
@@ -1911,9 +1912,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                     Context.Memory.Fill(GetDramAddressFromPa(dstFirstPagePa), unusedSizeBefore, (byte)_ipcFillValue);
 
                     ulong copySize = addressRounded <= endAddr ? addressRounded - address : size;
-                    var data = srcPageTable.GetSpan(addressTruncated + unusedSizeBefore, (int)copySize);
+                    var data = srcPageTable.GetReadOnlySequence(addressTruncated + unusedSizeBefore, (int)copySize);
 
-                    Context.Memory.Write(GetDramAddressFromPa(dstFirstPagePa + unusedSizeBefore), data);
+                    ((IWritableBlock)Context.Memory).Write(GetDramAddressFromPa(dstFirstPagePa + unusedSizeBefore), data);
 
                     firstPageFillAddress += unusedSizeBefore + copySize;
 
@@ -1977,9 +1978,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                 if (send)
                 {
                     ulong copySize = endAddr - endAddrTruncated;
-                    var data = srcPageTable.GetSpan(endAddrTruncated, (int)copySize);
+                    var data = srcPageTable.GetReadOnlySequence(endAddrTruncated, (int)copySize);
 
-                    Context.Memory.Write(GetDramAddressFromPa(dstLastPagePa), data);
+                    ((IWritableBlock)Context.Memory).Write(GetDramAddressFromPa(dstLastPagePa), data);
 
                     lastPageFillAddr += copySize;
 
@@ -2943,6 +2944,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         /// <param name="pageList">Page list where the ranges will be added</param>
         protected abstract void GetPhysicalRegions(ulong va, ulong size, KPageList pageList);
 
+        /// <summary>
+        /// Gets a read-only sequence of data from CPU mapped memory.
+        /// </summary>
+        /// <remarks>
+        /// Allows reading non-contiguous memory without first copying it to a newly allocated single contiguous block.
+        /// </remarks>
+        /// <param name="va">Virtual address of the data</param>
+        /// <param name="size">Size of the data</param>
+        /// <returns>A read-only sequence of the data</returns>
+        /// <exception cref="Ryujinx.Memory.InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        protected abstract ReadOnlySequence<byte> GetReadOnlySequence(ulong va, int size);
+
         /// <summary>
         /// Gets a read-only span of data from CPU mapped memory.
         /// </summary>
@@ -2952,7 +2965,6 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         /// </remarks>
         /// <param name="va">Virtual address of the data</param>
         /// <param name="size">Size of the data</param>
-        /// <param name="tracked">True if read tracking is triggered on the span</param>
         /// <returns>A read-only span of the data</returns>
         /// <exception cref="Ryujinx.Memory.InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
         protected abstract ReadOnlySpan<byte> GetSpan(ulong va, int size);
@@ -3060,6 +3072,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         /// <param name="size">Size of the region</param>
         protected abstract void SignalMemoryTracking(ulong va, ulong size, bool write);
 
+        /// <summary>
+        /// Writes data to CPU mapped memory, with write tracking.
+        /// </summary>
+        /// <param name="va">Virtual address to write the data into</param>
+        /// <param name="data">Data to be written</param>
+        /// <exception cref="Ryujinx.Memory.InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        protected abstract void Write(ulong va, ReadOnlySequence<byte> data);
+
         /// <summary>
         /// Writes data to CPU mapped memory, with write tracking.
         /// </summary>

src/Ryujinx.HLE/HOS/Services/Fatal/IService.cs

@@ -60,7 +60,7 @@ namespace Ryujinx.HLE.HOS.Services.Fatal
             errorReport.AppendLine($"\tResultCode: {((int)resultCode & 0x1FF) + 2000}-{((int)resultCode >> 9) & 0x3FFF:d4}");
             errorReport.AppendLine($"\tFatalPolicy: {fatalPolicy}");
 
-            if (cpuContext != null)
+            if (!cpuContext.IsEmpty)
             {
                 errorReport.AppendLine("CPU Context:");
 

src/Ryujinx.HLE/HOS/Services/SurfaceFlinger/IBinder.cs

@@ -13,10 +13,10 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 
         ResultCode OnTransact(uint code, uint flags, ReadOnlySpan<byte> inputParcel, Span<byte> outputParcel)
         {
-            Parcel inputParcelReader = new(inputParcel.ToArray());
+            using Parcel inputParcelReader = new(inputParcel);
 
             // TODO: support objects?
-            Parcel outputParcelWriter = new((uint)(outputParcel.Length - Unsafe.SizeOf<ParcelHeader>()), 0);
+            using Parcel outputParcelWriter = new((uint)(outputParcel.Length - Unsafe.SizeOf<ParcelHeader>()), 0);
 
             string inputInterfaceToken = inputParcelReader.ReadInterfaceToken();
 

src/Ryujinx.HLE/HOS/Services/SurfaceFlinger/Parcel.cs

@@ -1,7 +1,9 @@
 using Ryujinx.Common;
+using Ryujinx.Common.Memory;
 using Ryujinx.Common.Utilities;
 using Ryujinx.HLE.HOS.Services.SurfaceFlinger.Types;
 using System;
+using System.Buffers;
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -9,13 +11,13 @@ using System.Text;
 
 namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 {
-    class Parcel
+    sealed class Parcel : IDisposable
     {
-        private readonly byte[] _rawData;
+        private readonly IMemoryOwner<byte> _rawDataOwner;
 
-        private Span<byte> Raw => new(_rawData);
+        private Span<byte> Raw => _rawDataOwner.Memory.Span;
 
-        private ref ParcelHeader Header => ref MemoryMarshal.Cast<byte, ParcelHeader>(_rawData)[0];
+        private ref ParcelHeader Header => ref MemoryMarshal.Cast<byte, ParcelHeader>(Raw)[0];
 
         private Span<byte> Payload => Raw.Slice((int)Header.PayloadOffset, (int)Header.PayloadSize);
 
@@ -24,9 +26,11 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
         private int _payloadPosition;
         private int _objectPosition;
 
-        public Parcel(byte[] rawData)
+        private bool _isDisposed;
+
+        public Parcel(ReadOnlySpan<byte> data)
         {
-            _rawData = rawData;
+            _rawDataOwner = ByteMemoryPool.RentCopy(data);
 
             _payloadPosition = 0;
             _objectPosition = 0;
@@ -36,7 +40,7 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
         {
             uint headerSize = (uint)Unsafe.SizeOf<ParcelHeader>();
 
-            _rawData = new byte[BitUtils.AlignUp<uint>(headerSize + payloadSize + objectsSize, 4)];
+            _rawDataOwner = ByteMemoryPool.RentCleared(BitUtils.AlignUp<uint>(headerSize + payloadSize + objectsSize, 4));
 
             Header.PayloadSize = payloadSize;
             Header.ObjectsSize = objectsSize;
@@ -132,7 +136,9 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 
             // TODO: figure out what this value is
 
-            WriteInplaceObject(new byte[4] { 0, 0, 0, 0 });
+            Span<byte> fourBytes = stackalloc byte[4];
+
+            WriteInplaceObject(fourBytes);
         }
 
         public AndroidStrongPointer<T> ReadStrongPointer<T>() where T : unmanaged, IFlattenable
@@ -219,5 +225,15 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 
             return Raw[..(int)(Header.PayloadSize + Header.ObjectsSize + Unsafe.SizeOf<ParcelHeader>())];
         }
+
+        public void Dispose()
+        {
+            if (!_isDisposed)
+            {
+                _isDisposed = true;
+
+                _rawDataOwner.Dispose();
+            }
+        }
     }
 }

src/Ryujinx.HLE/HOS/Services/Vi/RootService/IApplicationDisplayService.cs

@@ -250,7 +250,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
 
             context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
-            Parcel parcel = new(0x28, 0x4);
+            using Parcel parcel = new(0x28, 0x4);
 
             parcel.WriteObject(producer, "dispdrv\0");
 
@@ -288,7 +288,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
 
             context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
-            Parcel parcel = new(0x28, 0x4);
+            using Parcel parcel = new(0x28, 0x4);
 
             parcel.WriteObject(producer, "dispdrv\0");
 

src/Ryujinx.HLE/HOS/TamperMachine.cs

@@ -143,7 +143,7 @@ namespace Ryujinx.HLE.HOS
 
             try
             {
-                ControllerKeys pressedKeys = (ControllerKeys)Thread.VolatileRead(ref _pressedKeys);
+                ControllerKeys pressedKeys = (ControllerKeys)Volatile.Read(ref _pressedKeys);
                 program.Process.TamperedCodeMemory = false;
                 program.Execute(pressedKeys);
 
@@ -175,14 +175,14 @@ namespace Ryujinx.HLE.HOS
             {
                 if (input.PlayerId == PlayerIndex.Player1 || input.PlayerId == PlayerIndex.Handheld)
                 {
-                    Thread.VolatileWrite(ref _pressedKeys, (long)input.Buttons);
+                    Volatile.Write(ref _pressedKeys, (long)input.Buttons);
 
                     return;
                 }
             }
 
             // Clear the input because player one is not conected.
-            Thread.VolatileWrite(ref _pressedKeys, 0);
+            Volatile.Write(ref _pressedKeys, 0);
         }
     }
 }

src/Ryujinx.Memory/IWritableBlock.cs

@@ -1,9 +1,25 @@
 using System;
+using System.Buffers;
 
 namespace Ryujinx.Memory
 {
     public interface IWritableBlock
     {
+        /// <summary>
+        /// Writes data to CPU mapped memory, with write tracking.
+        /// </summary>
+        /// <param name="va">Virtual address to write the data into</param>
+        /// <param name="data">Data to be written</param>
+        /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        void Write(ulong va, ReadOnlySequence<byte> data)
+        {
+            foreach (ReadOnlyMemory<byte> segment in data)
+            {
+                Write(va, segment.Span);
+                va += (ulong)segment.Length;
+            }
+        }
+
         void Write(ulong va, ReadOnlySpan<byte> data);
 
         void WriteUntracked(ulong va, ReadOnlySpan<byte> data) => Write(va, data);

src/Ryujinx.Tests/Memory/PartialUnmaps.cs

@@ -388,14 +388,14 @@ namespace Ryujinx.Tests.Memory
                     {
                         rwLock.AcquireReaderLock();
 
-                        int originalValue = Thread.VolatileRead(ref value);
+                        int originalValue = Volatile.Read(ref value);
 
                         count++;
 
                         // Spin a bit.
                         for (int i = 0; i < 100; i++)
                         {
-                            if (Thread.VolatileRead(ref readersAllowed) == 0)
+                            if (Volatile.Read(ref readersAllowed) == 0)
                             {
                                 error = true;
                                 running = false;
@@ -403,7 +403,7 @@ namespace Ryujinx.Tests.Memory
                         }
 
                         // Should not change while the lock is held.
-                        if (Thread.VolatileRead(ref value) != originalValue)
+                        if (Volatile.Read(ref value) != originalValue)
                         {
                             error = true;
                             running = false;