SetProcessMemoryPermission address and size are always 64-bit (#6977)

* JIT: Coalesce copies on LSRA with simple register preferencing

* PPTC version bump

Directory.Packages.props

@@ -22,7 +22,7 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
     <PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.6.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
-    <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
+    <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
     <PackageVersion Include="MsgPack.Cli" Version="1.0.1" />
     <PackageVersion Include="NetCoreServer" Version="8.0.7" />
     <PackageVersion Include="NUnit" Version="3.13.3" />

src/ARMeilleure/CodeGen/RegisterAllocators/LinearScanAllocator.cs

@@ -251,7 +251,20 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                 }
             }
 
-            int selectedReg = GetHighestValueIndex(freePositions);
+            // If this is a copy destination variable, we prefer the register used for the copy source.
+            // If the register is available, then the copy can be eliminated later as both source
+            // and destination will use the same register.
+            int selectedReg;
+
+            if (current.TryGetCopySourceRegister(out int preferredReg) && freePositions[preferredReg] >= current.GetEnd())
+            {
+                selectedReg = preferredReg;
+            }
+            else
+            {
+                selectedReg = GetHighestValueIndex(freePositions);
+            }
+
             int selectedNextUse = freePositions[selectedReg];
 
             // Intervals starts and ends at odd positions, unless they span an entire
@@ -431,7 +444,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             }
         }
 
-        private static int GetHighestValueIndex(Span<int> span)
+        private static int GetHighestValueIndex(ReadOnlySpan<int> span)
         {
             int highest = int.MinValue;
 
@@ -798,12 +811,12 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             // The "visited" state is stored in the MSB of the local's value.
             const ulong VisitedMask = 1ul << 63;
 
-            bool IsVisited(Operand local)
+            static bool IsVisited(Operand local)
             {
                 return (local.GetValueUnsafe() & VisitedMask) != 0;
             }
 
-            void SetVisited(Operand local)
+            static void SetVisited(Operand local)
             {
                 local.GetValueUnsafe() |= VisitedMask;
             }
@@ -826,9 +839,25 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
                         {
                             dest.NumberLocal(_intervals.Count);
 
-                            _intervals.Add(new LiveInterval(dest));
+                            LiveInterval interval = new LiveInterval(dest);
+                            _intervals.Add(interval);
 
                             SetVisited(dest);
+
+                            // If this is a copy (or copy-like operation), set the copy source interval as well.
+                            // This is used for register preferencing later on, which allows the copy to be eliminated
+                            // in some cases.
+                            if (node.Instruction == Instruction.Copy || node.Instruction == Instruction.ZeroExtend32)
+                            {
+                                Operand source = node.GetSource(0);
+
+                                if (source.Kind == OperandKind.LocalVariable &&
+                                    source.GetLocalNumber() > 0 &&
+                                    (node.Instruction == Instruction.Copy || source.Type == OperandType.I32))
+                                {
+                                    interval.SetCopySource(_intervals[source.GetLocalNumber()]);
+                                }
+                            }
                         }
                     }
                 }

src/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs

@@ -19,6 +19,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             public LiveRange CurrRange;
 
             public LiveInterval Parent;
+            public LiveInterval CopySource;
 
             public UseList Uses;
             public LiveIntervalList Children;
@@ -37,6 +38,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
         private ref LiveRange CurrRange => ref _data->CurrRange;
         private ref LiveRange PrevRange => ref _data->PrevRange;
         private ref LiveInterval Parent => ref _data->Parent;
+        private ref LiveInterval CopySource => ref _data->CopySource;
         private ref UseList Uses => ref _data->Uses;
         private ref LiveIntervalList Children => ref _data->Children;
 
@@ -78,6 +80,25 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
             Register = register;
         }
 
+        public void SetCopySource(LiveInterval copySource)
+        {
+            CopySource = copySource;
+        }
+
+        public bool TryGetCopySourceRegister(out int copySourceRegIndex)
+        {
+            if (CopySource._data != null)
+            {
+                copySourceRegIndex = CopySource.Register.Index;
+
+                return true;
+            }
+
+            copySourceRegIndex = 0;
+
+            return false;
+        }
+
         public void Reset()
         {
             PrevRange = default;

src/ARMeilleure/Translation/ControlFlowGraph.cs

@@ -11,7 +11,7 @@ namespace ARMeilleure.Translation
         private int[] _postOrderMap;
 
         public int LocalsCount { get; private set; }
-        public BasicBlock Entry { get; }
+        public BasicBlock Entry { get; private set; }
         public IntrusiveList<BasicBlock> Blocks { get; }
         public BasicBlock[] PostOrderBlocks => _postOrderBlocks;
         public int[] PostOrderMap => _postOrderMap;
@@ -34,6 +34,15 @@ namespace ARMeilleure.Translation
             return result;
         }
 
+        public void UpdateEntry(BasicBlock newEntry)
+        {
+            newEntry.AddSuccessor(Entry);
+
+            Entry = newEntry;
+            Blocks.AddFirst(newEntry);
+            Update();
+        }
+
         public void Update()
         {
             RemoveUnreachableBlocks(Blocks);

src/ARMeilleure/Translation/PTC/Ptc.cs

@@ -29,7 +29,7 @@ namespace ARMeilleure.Translation.PTC
         private const string OuterHeaderMagicString = "PTCohd\0\0";
         private const string InnerHeaderMagicString = "PTCihd\0\0";
 
-        private const uint InternalVersion = 6634; //! To be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 6950; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";

src/ARMeilleure/Translation/RegisterUsage.cs

@@ -89,6 +89,17 @@ namespace ARMeilleure.Translation
 
         public static void RunPass(ControlFlowGraph cfg, ExecutionMode mode)
         {
+            if (cfg.Entry.Predecessors.Count != 0)
+            {
+                // We expect the entry block to have no predecessors.
+                // This is required because we have a implicit context load at the start of the function,
+                // but if there is a jump to the start of the function, the context load would trash the modified values.
+                // Here we insert a new entry block that will jump to the existing entry block.
+                BasicBlock newEntry = new BasicBlock(cfg.Blocks.Count);
+
+                cfg.UpdateEntry(newEntry);
+            }
+
             // Compute local register inputs and outputs used inside blocks.
             RegisterMask[] localInputs = new RegisterMask[cfg.Blocks.Count];
             RegisterMask[] localOutputs = new RegisterMask[cfg.Blocks.Count];
@@ -201,7 +212,7 @@ namespace ARMeilleure.Translation
 
                 // The only block without any predecessor should be the entry block.
                 // It always needs a context load as it is the first block to run.
-                if (block.Predecessors.Count == 0 || hasContextLoad)
+                if (block == cfg.Entry || hasContextLoad)
                 {
                     long vecMask = globalInputs[block.Index].VecMask;
                     long intMask = globalInputs[block.Index].IntMask;

src/Ryujinx.Audio.Backends.SDL2/SDL2HardwareDeviceSession.cs

@@ -89,9 +89,9 @@ namespace Ryujinx.Audio.Backends.SDL2
                 return;
             }
 
-            using IMemoryOwner<byte> samplesOwner = ByteMemoryPool.Rent(frameCount * _bytesPerFrame);
+            using SpanOwner<byte> samplesOwner = SpanOwner<byte>.Rent(frameCount * _bytesPerFrame);
 
-            Span<byte> samples = samplesOwner.Memory.Span;
+            Span<byte> samples = samplesOwner.Span;
 
             _ringBuffer.Read(samples, 0, samples.Length);
 

src/Ryujinx.Audio.Backends.SoundIo/SoundIoHardwareDeviceSession.cs

@@ -122,9 +122,9 @@ namespace Ryujinx.Audio.Backends.SoundIo
 
             int channelCount = areas.Length;
 
-            using IMemoryOwner<byte> samplesOwner = ByteMemoryPool.Rent(frameCount * bytesPerFrame);
+            using SpanOwner<byte> samplesOwner = SpanOwner<byte>.Rent(frameCount * bytesPerFrame);
 
-            Span<byte> samples = samplesOwner.Memory.Span;
+            Span<byte> samples = samplesOwner.Span;
 
             _ringBuffer.Read(samples, 0, samples.Length);
 

src/Ryujinx.Audio/Backends/Common/DynamicRingBuffer.cs

@@ -14,7 +14,7 @@ namespace Ryujinx.Audio.Backends.Common
 
         private readonly object _lock = new();
 
-        private IMemoryOwner<byte> _bufferOwner;
+        private MemoryOwner<byte> _bufferOwner;
         private Memory<byte> _buffer;
         private int _size;
         private int _headOffset;
@@ -24,7 +24,7 @@ namespace Ryujinx.Audio.Backends.Common
 
         public DynamicRingBuffer(int initialCapacity = RingBufferAlignment)
         {
-            _bufferOwner = ByteMemoryPool.RentCleared(initialCapacity);
+            _bufferOwner = MemoryOwner<byte>.RentCleared(initialCapacity);
             _buffer = _bufferOwner.Memory;
         }
 
@@ -62,7 +62,7 @@ namespace Ryujinx.Audio.Backends.Common
 
         private void SetCapacityLocked(int capacity)
         {
-            IMemoryOwner<byte> newBufferOwner = ByteMemoryPool.RentCleared(capacity);
+            MemoryOwner<byte> newBufferOwner = MemoryOwner<byte>.RentCleared(capacity);
             Memory<byte> newBuffer = newBufferOwner.Memory;
 
             if (_size > 0)

src/Ryujinx.Common/Memory/MemoryOwner.cs

@@ -124,7 +124,7 @@ namespace Ryujinx.Common.Memory
 
             if (array is not null)
             {
-                ArrayPool<T>.Shared.Return(array);
+                ArrayPool<T>.Shared.Return(array, RuntimeHelpers.IsReferenceOrContainsReferences<T>());
             }
         }
 

src/Ryujinx.Common/Memory/SpanOwner.cs

@@ -108,7 +108,7 @@ namespace Ryujinx.Common.Memory
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void Dispose()
         {
-            ArrayPool<T>.Shared.Return(_array);
+            ArrayPool<T>.Shared.Return(_array, RuntimeHelpers.IsReferenceOrContainsReferences<T>());
         }
     }
 }

src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs

@@ -22,7 +22,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
         private const ushort FileFormatVersionMajor = 1;
         private const ushort FileFormatVersionMinor = 2;
         private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor;
-        private const uint CodeGenVersion = 6852;
+        private const uint CodeGenVersion = 6921;
 
         private const string SharedTocFileName = "shared.toc";
         private const string SharedDataFileName = "shared.data";

src/Ryujinx.Graphics.Shader/Instructions/InstEmitPredicate.cs

@@ -24,7 +24,7 @@ namespace Ryujinx.Graphics.Shader.Instructions
 
             if (op.BVal)
             {
-                context.Copy(dest, context.ConditionalSelect(res, ConstF(1), Const(0)));
+                context.Copy(dest, context.ConditionalSelect(res, ConstF(1), ConstF(0)));
             }
             else
             {

src/Ryujinx.Graphics.Shader/IntermediateRepresentation/Instruction.cs

@@ -156,6 +156,26 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
             return false;
         }
 
+        public static bool IsComparison(this Instruction inst)
+        {
+            switch (inst & Instruction.Mask)
+            {
+                case Instruction.CompareEqual:
+                case Instruction.CompareGreater:
+                case Instruction.CompareGreaterOrEqual:
+                case Instruction.CompareGreaterOrEqualU32:
+                case Instruction.CompareGreaterU32:
+                case Instruction.CompareLess:
+                case Instruction.CompareLessOrEqual:
+                case Instruction.CompareLessOrEqualU32:
+                case Instruction.CompareLessU32:
+                case Instruction.CompareNotEqual:
+                    return true;
+            }
+
+            return false;
+        }
+
         public static bool IsTextureQuery(this Instruction inst)
         {
             inst &= Instruction.Mask;

src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs

@@ -141,16 +141,16 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
             return true;
         }
 
-        private static bool IsBindlessAccessAllowed(Operand nvHandle)
+        private static bool IsBindlessAccessAllowed(Operand bindlessHandle)
         {
-            if (nvHandle.Type == OperandType.ConstantBuffer)
+            if (bindlessHandle.Type == OperandType.ConstantBuffer)
             {
                 // Bindless access with handles from constant buffer is allowed.
 
                 return true;
             }
 
-            if (nvHandle.AsgOp is not Operation handleOp ||
+            if (bindlessHandle.AsgOp is not Operation handleOp ||
                 handleOp.Inst != Instruction.Load ||
                 (handleOp.StorageKind != StorageKind.Input && handleOp.StorageKind != StorageKind.StorageBuffer))
             {
@@ -300,7 +300,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
                         resourceManager,
                         gpuAccessor,
                         texOp,
-                        TextureHandle.PackOffsets(src0.GetCbufOffset(), ((src1.Value >> 20) & 0xfff), handleType),
+                        TextureHandle.PackOffsets(src0.GetCbufOffset(), (src1.Value >> 20) & 0xfff, handleType),
                         TextureHandle.PackSlots(src0.GetCbufSlot(), 0),
                         rewriteSamplerType,
                         isImage: false);

src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessToArray.cs

@@ -126,7 +126,9 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
                     continue;
                 }
 
-                if (texOp.GetSource(0).AsgOp is not Operation handleAsgOp)
+                Operand bindlessHandle = Utils.FindLastOperation(texOp.GetSource(0), block);
+
+                if (bindlessHandle.AsgOp is not Operation handleAsgOp)
                 {
                     continue;
                 }
@@ -137,8 +139,8 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
                 if (handleAsgOp.Inst == Instruction.BitwiseOr)
                 {
-                    Operand src0 = handleAsgOp.GetSource(0);
-                    Operand src1 = handleAsgOp.GetSource(1);
+                    Operand src0 = Utils.FindLastOperation(handleAsgOp.GetSource(0), block);
+                    Operand src1 = Utils.FindLastOperation(handleAsgOp.GetSource(1), block);
 
                     if (src0.Type == OperandType.ConstantBuffer && src1.AsgOp is Operation)
                     {

src/Ryujinx.Graphics.Shader/Translation/Optimizations/Optimizer.cs

@@ -152,18 +152,14 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
         {
             // If all phi sources are the same, we can propagate it and remove the phi.
 
-            Operand firstSrc = phi.GetSource(0);
-
-            for (int index = 1; index < phi.SourcesCount; index++)
+            if (!Utils.AreAllSourcesTheSameOperand(phi))
             {
-                if (!IsSameOperand(firstSrc, phi.GetSource(index)))
-                {
-                    return false;
-                }
+                return false;
             }
 
             // All sources are equal, we can propagate the value.
 
+            Operand firstSrc = phi.GetSource(0);
             Operand dest = phi.Dest;
 
             INode[] uses = dest.UseOps.ToArray();
@@ -182,17 +178,6 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
             return true;
         }
 
-        private static bool IsSameOperand(Operand x, Operand y)
-        {
-            if (x.Type != y.Type || x.Value != y.Value)
-            {
-                return false;
-            }
-
-            // TODO: Handle Load operations with the same storage and the same constant parameters.
-            return x.Type == OperandType.Constant || x.Type == OperandType.ConstantBuffer;
-        }
-
         private static bool PropagatePack(Operation packOp)
         {
             // Propagate pack source operands to uses by unpack

src/Ryujinx.Graphics.Shader/Translation/Optimizations/Simplification.cs

@@ -31,6 +31,10 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
                     TryEliminateBitwiseOr(operation);
                     break;
 
+                case Instruction.CompareNotEqual:
+                    TryEliminateCompareNotEqual(operation);
+                    break;
+
                 case Instruction.ConditionalSelect:
                     TryEliminateConditionalSelect(operation);
                     break;
@@ -174,6 +178,32 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
             }
         }
 
+        private static void TryEliminateCompareNotEqual(Operation operation)
+        {
+            // Comparison instruction returns 0 if the result is false, and -1 if true.
+            // Doing a not equal zero comparison on the result is redundant, so we can just copy the first result in this case.
+
+            Operand lhs = operation.GetSource(0);
+            Operand rhs = operation.GetSource(1);
+
+            if (lhs.Type == OperandType.Constant)
+            {
+                (lhs, rhs) = (rhs, lhs);
+            }
+
+            if (rhs.Type != OperandType.Constant || rhs.Value != 0)
+            {
+                return;
+            }
+
+            if (lhs.AsgOp is not Operation compareOp || !compareOp.Inst.IsComparison())
+            {
+                return;
+            }
+
+            operation.TurnIntoCopy(lhs);
+        }
+
         private static void TryEliminateConditionalSelect(Operation operation)
         {
             Operand cond = operation.GetSource(0);

src/Ryujinx.Graphics.Shader/Translation/Optimizations/Utils.cs

@@ -34,6 +34,50 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
             return elemIndexSrc.Type == OperandType.Constant && elemIndexSrc.Value == elemIndex;
         }
 
+        private static bool IsSameOperand(Operand x, Operand y)
+        {
+            if (x.Type != y.Type || x.Value != y.Value)
+            {
+                return false;
+            }
+
+            // TODO: Handle Load operations with the same storage and the same constant parameters.
+            return x == y || x.Type == OperandType.Constant || x.Type == OperandType.ConstantBuffer;
+        }
+
+        private static bool AreAllSourcesEqual(INode node, INode otherNode)
+        {
+            if (node.SourcesCount != otherNode.SourcesCount)
+            {
+                return false;
+            }
+
+            for (int index = 0; index < node.SourcesCount; index++)
+            {
+                if (!IsSameOperand(node.GetSource(index), otherNode.GetSource(index)))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        public static bool AreAllSourcesTheSameOperand(INode node)
+        {
+            Operand firstSrc = node.GetSource(0);
+
+            for (int index = 1; index < node.SourcesCount; index++)
+            {
+                if (!IsSameOperand(firstSrc, node.GetSource(index)))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
         private static Operation FindBranchSource(BasicBlock block)
         {
             foreach (BasicBlock sourceBlock in block.Predecessors)
@@ -55,6 +99,19 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
             return inst == Instruction.BranchIfFalse || inst == Instruction.BranchIfTrue;
         }
 
+        private static bool IsSameCondition(Operand currentCondition, Operand queryCondition)
+        {
+            if (currentCondition == queryCondition)
+            {
+                return true;
+            }
+
+            return currentCondition.AsgOp is Operation currentOperation &&
+                queryCondition.AsgOp is Operation queryOperation &&
+                currentOperation.Inst == queryOperation.Inst &&
+                AreAllSourcesEqual(currentOperation, queryOperation);
+        }
+
         private static bool BlockConditionsMatch(BasicBlock currentBlock, BasicBlock queryBlock)
         {
             // Check if all the conditions for the query block are satisfied by the current block.
@@ -70,10 +127,10 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
             return currentBranch != null && queryBranch != null &&
                    currentBranch.Inst == queryBranch.Inst &&
-                   currentCondition == queryCondition;
+                   IsSameCondition(currentCondition, queryCondition);
         }
 
-        public static Operand FindLastOperation(Operand source, BasicBlock block)
+        public static Operand FindLastOperation(Operand source, BasicBlock block, bool recurse = true)
         {
             if (source.AsgOp is PhiNode phiNode)
             {
@@ -84,10 +141,23 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
                 for (int i = phiNode.SourcesCount - 1; i >= 0; i--)
                 {
                     BasicBlock phiBlock = phiNode.GetBlock(i);
+                    Operand phiSource = phiNode.GetSource(i);
 
                     if (BlockConditionsMatch(block, phiBlock))
                     {
-                        return phiNode.GetSource(i);
+                        return phiSource;
+                    }
+                    else if (recurse && phiSource.AsgOp is PhiNode)
+                    {
+                        // Phi source is another phi.
+                        // Let's check if that phi has a block that matches our condition.
+
+                        Operand match = FindLastOperation(phiSource, block, false);
+
+                        if (match != phiSource)
+                        {
+                            return match;
+                        }
                     }
                 }
             }

src/Ryujinx.HLE/HOS/Kernel/SupervisorCall/Syscall.cs

@@ -616,7 +616,7 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
                 }
             }
 
-            ArrayPool<KSynchronizationObject>.Shared.Return(syncObjsArray);
+            ArrayPool<KSynchronizationObject>.Shared.Return(syncObjsArray, true);
 
             return result;
         }
@@ -1546,8 +1546,8 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
 #pragma warning disable CA1822 // Mark member as static
         public Result SetProcessMemoryPermission(
             int handle,
-            [PointerSized] ulong src,
-            [PointerSized] ulong size,
+            ulong src,
+            ulong size,
             KMemoryPermission permission)
         {
             if (!PageAligned(src))

src/Ryujinx.HLE/HOS/Kernel/Threading/KSynchronization.cs

@@ -104,7 +104,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
                     }
                 }
 
-                ArrayPool<LinkedListNode<KThread>>.Shared.Return(syncNodesArray);
+                ArrayPool<LinkedListNode<KThread>>.Shared.Return(syncNodesArray, true);
             }
 
             _context.CriticalSection.Leave();