Add support to IVirtualMemoryManager for zero-copy reads (#6251)
* - WritableRegion: enable wrapping IMemoryOwner<byte> - IVirtualMemoryManager impls of GetWritableRegion() use pooled memory when region is non-contiguous. - IVirtualMemoryManager: add GetReadOnlySequence() and impls - ByteMemoryPool: add new method RentCopy() - ByteMemoryPool: make class static, remove ctor and singleton field from earlier impl * - BytesReadOnlySequenceSegment: move from Ryujinx.Common.Memory to Ryujinx.Memory - BytesReadOnlySequenceSegment: add IsContiguousWith() and Replace() methods - VirtualMemoryManagerBase: - remove generic type parameters, instead use ulong for virtual addresses and nuint for host/physical addresses - implement IWritableBlock - add virtual GetReadOnlySequence() with coalescing of contiguous segments - add virtual GetSpan() - add virtual GetWritableRegion() - add abstract IsMapped() - add virtual MapForeign(ulong, nuint, ulong) - add virtual Read<T>() - add virtual Read(ulong, Span<byte>) - add virtual ReadTracked<T>() - add virtual SignalMemoryTracking() - add virtual Write() - add virtual Write<T>() - add virtual WriteUntracked() - add virtual WriteWithRedundancyCheck() - VirtualMemoryManagerRefCountedBase: remove generic type parameters - AddressSpaceManager: remove redundant methods, add required overrides - HvMemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManagerHostMapped: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - NativeMemoryManager: add get properties for Pointer and Length - throughout: removed invalid <inheritdoc/> comments * make HvMemoryManager class sealed * remove unused method * adjust MemoryManagerHostTracked * let MemoryManagerHostTracked override WriteImpl()
This commit is contained in:
jhorv
GitHub
@ -3,7 +3,6 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Ryujinx.Memory
|
||||
{
|
||||
@ -11,7 +10,7 @@ namespace Ryujinx.Memory
|
||||
/// Represents a address space manager.
|
||||
/// Supports virtual memory region mapping, address translation and read/write access to mapped regions.
|
||||
/// </summary>
|
||||
public sealed class AddressSpaceManager : VirtualMemoryManagerBase<ulong, nuint>, IVirtualMemoryManager, IWritableBlock
|
||||
public sealed class AddressSpaceManager : VirtualMemoryManagerBase, IVirtualMemoryManager
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public bool Supports4KBPages => true;
|
||||
@ -63,8 +62,7 @@ namespace Ryujinx.Memory
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void MapForeign(ulong va, nuint hostPointer, ulong size)
|
||||
public override void MapForeign(ulong va, nuint hostPointer, ulong size)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
@ -92,106 +90,6 @@ namespace Ryujinx.Memory
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public T Read<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write<T>(ulong va, T value) where T : unmanaged
|
||||
{
|
||||
Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
AssertValidAddressAndSize(va, (ulong)data.Length);
|
||||
|
||||
if (IsContiguousAndMapped(va, data.Length))
|
||||
{
|
||||
data.CopyTo(GetHostSpanContiguous(va, data.Length));
|
||||
}
|
||||
else
|
||||
{
|
||||
int offset = 0, size;
|
||||
|
||||
if ((va & PageMask) != 0)
|
||||
{
|
||||
size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
|
||||
|
||||
data[..size].CopyTo(GetHostSpanContiguous(va, size));
|
||||
|
||||
offset += size;
|
||||
}
|
||||
|
||||
for (; offset < data.Length; offset += size)
|
||||
{
|
||||
size = Math.Min(data.Length - offset, PageSize);
|
||||
|
||||
data.Slice(offset, size).CopyTo(GetHostSpanContiguous(va + (ulong)offset, size));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
Write(va, data);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return ReadOnlySpan<byte>.Empty;
|
||||
}
|
||||
|
||||
if (IsContiguousAndMapped(va, size))
|
||||
{
|
||||
return GetHostSpanContiguous(va, size);
|
||||
}
|
||||
else
|
||||
{
|
||||
Span<byte> data = new byte[size];
|
||||
|
||||
Read(va, data);
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return new WritableRegion(null, va, Memory<byte>.Empty);
|
||||
}
|
||||
|
||||
if (IsContiguousAndMapped(va, size))
|
||||
{
|
||||
return new WritableRegion(null, va, new NativeMemoryManager<byte>((byte*)GetHostAddress(va), size).Memory);
|
||||
}
|
||||
else
|
||||
{
|
||||
Memory<byte> memory = new byte[size];
|
||||
|
||||
GetSpan(va, size).CopyTo(memory.Span);
|
||||
|
||||
return new WritableRegion(this, va, memory);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public unsafe ref T GetRef<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
@ -203,50 +101,6 @@ namespace Ryujinx.Memory
|
||||
return ref *(T*)GetHostAddress(va);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetPagesCount(ulong va, uint size, out ulong startVa)
|
||||
{
|
||||
// WARNING: Always check if ulong does not overflow during the operations.
|
||||
startVa = va & ~(ulong)PageMask;
|
||||
ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
|
||||
|
||||
return (int)(vaSpan / PageSize);
|
||||
}
|
||||
|
||||
private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsContiguous(ulong va, int size)
|
||||
{
|
||||
if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
|
||||
for (int page = 0; page < pages - 1; page++)
|
||||
{
|
||||
if (!ValidateAddress(va + PageSize))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (GetHostAddress(va) + PageSize != GetHostAddress(va + PageSize))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
va += PageSize;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
|
||||
{
|
||||
@ -304,7 +158,7 @@ namespace Ryujinx.Memory
|
||||
return null;
|
||||
}
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
int pages = GetPagesCount(va, size, out va);
|
||||
|
||||
var regions = new List<HostMemoryRange>();
|
||||
|
||||
@ -336,9 +190,8 @@ namespace Ryujinx.Memory
|
||||
return regions;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMapped(ulong va)
|
||||
public override bool IsMapped(ulong va)
|
||||
{
|
||||
if (!ValidateAddress(va))
|
||||
{
|
||||
@ -351,7 +204,7 @@ namespace Ryujinx.Memory
|
||||
/// <inheritdoc/>
|
||||
public bool IsRangeMapped(ulong va, ulong size)
|
||||
{
|
||||
if (size == 0UL)
|
||||
if (size == 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
@ -376,11 +229,6 @@ namespace Ryujinx.Memory
|
||||
return true;
|
||||
}
|
||||
|
||||
private unsafe Span<byte> GetHostSpanContiguous(ulong va, int size)
|
||||
{
|
||||
return new Span<byte>((void*)GetHostAddress(va), size);
|
||||
}
|
||||
|
||||
private nuint GetHostAddress(ulong va)
|
||||
{
|
||||
return _pageTable.Read(va) + (nuint)(va & PageMask);
|
||||
@ -397,16 +245,16 @@ namespace Ryujinx.Memory
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
|
||||
{
|
||||
// Only the ARM Memory Manager has tracking for now.
|
||||
}
|
||||
protected unsafe override Memory<byte> GetPhysicalAddressMemory(nuint pa, int size)
|
||||
=> new NativeMemoryManager<byte>((byte*)pa, size).Memory;
|
||||
|
||||
protected override unsafe Span<byte> GetPhysicalAddressSpan(nuint pa, int size)
|
||||
=> new((void*)pa, size);
|
||||
=> new Span<byte>((void*)pa, size);
|
||||
|
||||
protected override nuint TranslateVirtualAddressForRead(ulong va)
|
||||
protected override nuint TranslateVirtualAddressChecked(ulong va)
|
||||
=> GetHostAddress(va);
|
||||
|
||||
protected override nuint TranslateVirtualAddressUnchecked(ulong va)
|
||||
=> GetHostAddress(va);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user