Compare commits

..

4 Commits

Author SHA1 Message Date
2c94ac455e GPU: Keep rendered textures without any pool references alive (#4662)
* GPU: Keep sampled textures without any pool references alive

Occasionally games are very wasteful and clear/write to a texture without ever sampling it. As rendered textures in NVN games seem to all have overlapping memory ranges, the texture will eventually get overwritten.

Normally, this would trigger a removal from the auto delete cache, but a pool entry would keep the texture alive. However, with these textures that are never used, they will get deleted immediately and recreated on the next frame.

This change makes it so the ShortTextureCache can keep textures that have naver had a pool reference alive for a few frames, so they're not constantly being created and deleted.

This improves performance in Zelda BOTW a little.

* Cleanup
2023-05-01 16:27:51 -03:00
e18d258fa0 GPU: Pre-emptively flush textures that are flushed often (to imported memory when available) (#4711)
* WIP texture pre-flush

Improve performance of TextureView GetData to buffer

Fix copy/sync ordering

Fix minor bug

Make this actually work

WIP host mapping stuff

* Fix usage flags

* message

* Cleanup 1

* Fix rebase

* Fix

* Improve pre-flush rules

* Fix pre-flush

* A lot of cleanup

* Use the host memory bits

* Select the correct memory type

* Cleanup TextureGroupHandle

* Missing comment

* Remove debugging logs

* Revert BufferHandle _value access modifier

* One interrupt action at a time.

* Support D32S8 to D24S8 conversion, safeguards

* Interrupt cannot happen in sync handle's lock

Waitable needs to be checked twice now, but this should stop it from deadlocking.

* Remove unused using

* Address some feedback

* Address feedback

* Address more feedback

* Address more feedback

* Improve sync rules

Should allow for faster sync in some cases.
2023-05-01 16:05:12 -03:00
36f10df775 GPU: Fix errors handling texture remapping (#4745)
* GPU: Fix errors handling texture remapping

- Fixes an error where a pool entry and memory mapping changing at the same time could cause a texture to rebind its data from the wrong GPU VA (data swaps)
- Fixes an error where the texture pool could act on a mapping change before the mapping has actually been changed ("Unmapped" event happens before change, we need to signal it changed _after_ it completes)

TODO: remove textures from partially mapped list... if they aren't.

* Add Remap actions for handling post-mapping behaviours

* Remove unused code.

* Address feedback

* Nit
2023-05-01 15:32:32 -03:00
680e548022 Uneven frame pacing with vsync (#4744)
fixes issue #3906
2023-04-29 21:54:41 +01:00
48 changed files with 1539 additions and 103 deletions

View File

@ -0,0 +1,8 @@
namespace Ryujinx.Graphics.GAL
{
public enum BufferAccess
{
Default,
FlushPersistent
}
}

View File

@ -21,11 +21,14 @@ namespace Ryujinx.Graphics.GAL
{
return CreateBuffer(size, BufferHandle.Null);
}
BufferHandle CreateBuffer(nint pointer, int size);
BufferHandle CreateBuffer(int size, BufferAccess access);
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);
ISampler CreateSampler(SamplerCreateInfo info);
ITexture CreateTexture(TextureCreateInfo info, float scale);
bool PrepareHostMapping(nint address, ulong size);
void CreateSync(ulong id, bool strict);

View File

@ -12,6 +12,7 @@ namespace Ryujinx.Graphics.GAL
void CopyTo(ITexture destination, int firstLayer, int firstLevel);
void CopyTo(ITexture destination, int srcLayer, int dstLayer, int srcLevel, int dstLevel);
void CopyTo(ITexture destination, Extents2D srcRegion, Extents2D dstRegion, bool linearFilter);
void CopyTo(BufferRange range, int layer, int level, int stride);
ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel);

View File

@ -43,6 +43,8 @@ namespace Ryujinx.Graphics.GAL.Multithreading
Register<ActionCommand>(CommandType.Action);
Register<CreateBufferCommand>(CommandType.CreateBuffer);
Register<CreateBufferAccessCommand>(CommandType.CreateBufferAccess);
Register<CreateHostBufferCommand>(CommandType.CreateHostBuffer);
Register<CreateProgramCommand>(CommandType.CreateProgram);
Register<CreateSamplerCommand>(CommandType.CreateSampler);
Register<CreateSyncCommand>(CommandType.CreateSync);
@ -69,6 +71,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
Register<TextureCopyToCommand>(CommandType.TextureCopyTo);
Register<TextureCopyToScaledCommand>(CommandType.TextureCopyToScaled);
Register<TextureCopyToSliceCommand>(CommandType.TextureCopyToSlice);
Register<TextureCopyToBufferCommand>(CommandType.TextureCopyToBuffer);
Register<TextureCreateViewCommand>(CommandType.TextureCreateView);
Register<TextureGetDataCommand>(CommandType.TextureGetData);
Register<TextureGetDataSliceCommand>(CommandType.TextureGetDataSlice);

View File

@ -4,6 +4,8 @@
{
Action,
CreateBuffer,
CreateBufferAccess,
CreateHostBuffer,
CreateProgram,
CreateSampler,
CreateSync,
@ -29,6 +31,7 @@
SamplerDispose,
TextureCopyTo,
TextureCopyToBuffer,
TextureCopyToScaled,
TextureCopyToSlice,
TextureCreateView,

View File

@ -0,0 +1,22 @@
namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
{
struct CreateBufferAccessCommand : IGALCommand, IGALCommand<CreateBufferAccessCommand>
{
public CommandType CommandType => CommandType.CreateBufferAccess;
private BufferHandle _threadedHandle;
private int _size;
private BufferAccess _access;
public void Set(BufferHandle threadedHandle, int size, BufferAccess access)
{
_threadedHandle = threadedHandle;
_size = size;
_access = access;
}
public static void Run(ref CreateBufferAccessCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, command._access));
}
}
}

View File

@ -0,0 +1,22 @@
namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
{
struct CreateHostBufferCommand : IGALCommand, IGALCommand<CreateHostBufferCommand>
{
public CommandType CommandType => CommandType.CreateHostBuffer;
private BufferHandle _threadedHandle;
private nint _pointer;
private int _size;
public void Set(BufferHandle threadedHandle, nint pointer, int size)
{
_threadedHandle = threadedHandle;
_pointer = pointer;
_size = size;
}
public static void Run(ref CreateHostBufferCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._pointer, command._size));
}
}
}

View File

@ -0,0 +1,29 @@
using Ryujinx.Graphics.GAL.Multithreading.Model;
using Ryujinx.Graphics.GAL.Multithreading.Resources;
namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture
{
struct TextureCopyToBufferCommand : IGALCommand, IGALCommand<TextureCopyToBufferCommand>
{
public CommandType CommandType => CommandType.TextureCopyToBuffer;
private TableRef<ThreadedTexture> _texture;
private BufferRange _range;
private int _layer;
private int _level;
private int _stride;
public void Set(TableRef<ThreadedTexture> texture, BufferRange range, int layer, int level, int stride)
{
_texture = texture;
_range = range;
_layer = layer;
_level = level;
_stride = stride;
}
public static void Run(ref TextureCopyToBufferCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
command._texture.Get(threaded).Base.CopyTo(threaded.Buffers.MapBufferRange(command._range), command._layer, command._level, command._stride);
}
}
}

View File

@ -108,6 +108,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
}
}
public void CopyTo(BufferRange range, int layer, int level, int stride)
{
_renderer.New<TextureCopyToBufferCommand>().Set(Ref(this), range, layer, level, stride);
_renderer.QueueCommand();
}
public void SetData(SpanOrArray<byte> data)
{
_renderer.New<TextureSetDataCommand>().Set(Ref(this), Ref(data.ToArray()));

View File

@ -57,6 +57,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
private int _refConsumerPtr;
private Action _interruptAction;
private object _interruptLock = new();
public event EventHandler<ScreenCaptureImageInfo> ScreenCaptured;
@ -274,6 +275,24 @@ namespace Ryujinx.Graphics.GAL.Multithreading
return handle;
}
public BufferHandle CreateBuffer(nint pointer, int size)
{
BufferHandle handle = Buffers.CreateBufferHandle();
New<CreateHostBufferCommand>().Set(handle, pointer, size);
QueueCommand();
return handle;
}
public BufferHandle CreateBuffer(int size, BufferAccess access)
{
BufferHandle handle = Buffers.CreateBufferHandle();
New<CreateBufferAccessCommand>().Set(handle, size, access);
QueueCommand();
return handle;
}
public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info)
{
var program = new ThreadedProgram(this);
@ -448,11 +467,14 @@ namespace Ryujinx.Graphics.GAL.Multithreading
}
else
{
while (Interlocked.CompareExchange(ref _interruptAction, action, null) != null) { }
lock (_interruptLock)
{
while (Interlocked.CompareExchange(ref _interruptAction, action, null) != null) { }
_galWorkAvailable.Set();
_galWorkAvailable.Set();
_interruptRun.WaitOne();
_interruptRun.WaitOne();
}
}
}
@ -461,6 +483,11 @@ namespace Ryujinx.Graphics.GAL.Multithreading
// Threaded renderer ignores given interrupt action, as it provides its own to the child renderer.
}
public bool PrepareHostMapping(nint address, ulong size)
{
return _baseRenderer.PrepareHostMapping(address, size);
}
public void Dispose()
{
// Dispose must happen from the render thread, after all commands have completed.

View File

@ -1,5 +1,6 @@
using Ryujinx.Graphics.Device;
using Ryujinx.Graphics.Gpu.Engine.MME;
using Ryujinx.Graphics.Gpu.Synchronization;
using System;
using System.Collections.Generic;
using System.Threading;
@ -59,7 +60,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo
if (_createSyncPending)
{
_createSyncPending = false;
_context.CreateHostSyncIfNeeded(false, false);
_context.CreateHostSyncIfNeeded(HostSyncFlags.None);
}
}
@ -157,7 +158,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo
}
else if (operation == SyncpointbOperation.Incr)
{
_context.CreateHostSyncIfNeeded(true, true);
_context.CreateHostSyncIfNeeded(HostSyncFlags.StrictSyncpoint);
_context.Synchronization.IncrementSyncpoint(syncpointId);
}
@ -184,7 +185,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo
{
_context.Renderer.Pipeline.CommandBufferBarrier();
_context.CreateHostSyncIfNeeded(false, true);
_context.CreateHostSyncIfNeeded(HostSyncFlags.Strict);
}
/// <summary>

View File

@ -252,6 +252,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo
public void Interrupt()
{
_interrupt = true;
_event.Set();
}
/// <summary>

View File

@ -3,6 +3,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Engine.GPFifo;
using Ryujinx.Graphics.Gpu.Engine.InlineToMemory;
using Ryujinx.Graphics.Gpu.Engine.Threed.Blender;
using Ryujinx.Graphics.Gpu.Synchronization;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
@ -254,7 +255,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
uint syncpointId = (uint)argument & 0xFFFF;
_context.AdvanceSequence();
_context.CreateHostSyncIfNeeded(true, true);
_context.CreateHostSyncIfNeeded(HostSyncFlags.StrictSyncpoint);
_context.Renderer.UpdateCounters(); // Poll the query counters, the game may want an updated result.
_context.Synchronization.IncrementSyncpoint(syncpointId);
}

View File

@ -60,14 +60,14 @@ namespace Ryujinx.Graphics.Gpu
/// If there are more than 0 items when this happens, a host sync object will be generated for the given <see cref="SyncNumber"/>,
/// and the SyncNumber will be incremented.
/// </summary>
internal List<Action> SyncActions { get; }
internal List<ISyncActionHandler> SyncActions { get; }
/// <summary>
/// Actions to be performed when a CPU waiting syncpoint is triggered.
/// If there are more than 0 items when this happens, a host sync object will be generated for the given <see cref="SyncNumber"/>,
/// and the SyncNumber will be incremented.
/// </summary>
internal List<Action> SyncpointActions { get; }
internal List<ISyncActionHandler> SyncpointActions { get; }
/// <summary>
/// Buffer migrations that are currently in-flight. These are checked whenever sync is created to determine if buffer migration
@ -114,8 +114,8 @@ namespace Ryujinx.Graphics.Gpu
HostInitalized = new ManualResetEvent(false);
SyncActions = new List<Action>();
SyncpointActions = new List<Action>();
SyncActions = new List<ISyncActionHandler>();
SyncpointActions = new List<ISyncActionHandler>();
BufferMigrations = new List<BufferMigration>();
DeferredActions = new Queue<Action>();
@ -296,9 +296,9 @@ namespace Ryujinx.Graphics.Gpu
/// Registers an action to be performed the next time a syncpoint is incremented.
/// This will also ensure a host sync object is created, and <see cref="SyncNumber"/> is incremented.
/// </summary>
/// <param name="action">The action to be performed on sync object creation</param>
/// <param name="action">The resource with action to be performed on sync object creation</param>
/// <param name="syncpointOnly">True if the sync action should only run when syncpoints are incremented</param>
public void RegisterSyncAction(Action action, bool syncpointOnly = false)
internal void RegisterSyncAction(ISyncActionHandler action, bool syncpointOnly = false)
{
if (syncpointOnly)
{
@ -315,10 +315,13 @@ namespace Ryujinx.Graphics.Gpu
/// Creates a host sync object if there are any pending sync actions. The actions will then be called.
/// If no actions are present, a host sync object is not created.
/// </summary>
/// <param name="syncpoint">True if host sync is being created by a syncpoint</param>
/// <param name="strict">True if the sync should signal as soon as possible</param>
public void CreateHostSyncIfNeeded(bool syncpoint, bool strict)
/// <param name="flags">Modifiers for how host sync should be created</param>
internal void CreateHostSyncIfNeeded(HostSyncFlags flags)
{
bool syncpoint = flags.HasFlag(HostSyncFlags.Syncpoint);
bool strict = flags.HasFlag(HostSyncFlags.Strict);
bool force = flags.HasFlag(HostSyncFlags.Force);
if (BufferMigrations.Count > 0)
{
ulong currentSyncNumber = Renderer.GetCurrentSync();
@ -336,24 +339,17 @@ namespace Ryujinx.Graphics.Gpu
}
}
if (_pendingSync || (syncpoint && SyncpointActions.Count > 0))
if (force || _pendingSync || (syncpoint && SyncpointActions.Count > 0))
{
Renderer.CreateSync(SyncNumber, strict);
SyncActions.ForEach(action => action.SyncPreAction(syncpoint));
SyncpointActions.ForEach(action => action.SyncPreAction(syncpoint));
SyncNumber++;
foreach (Action action in SyncActions)
{
action();
}
foreach (Action action in SyncpointActions)
{
action();
}
SyncActions.Clear();
SyncpointActions.Clear();
SyncActions.RemoveAll(action => action.SyncAction(syncpoint));
SyncpointActions.RemoveAll(action => action.SyncAction(syncpoint));
}
_pendingSync = false;

View File

@ -1,5 +1,4 @@
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
namespace Ryujinx.Graphics.Gpu.Image
@ -9,6 +8,7 @@ namespace Ryujinx.Graphics.Gpu.Image
/// </summary>
class ShortTextureCacheEntry
{
public bool IsAutoDelete;
public readonly TextureDescriptor Descriptor;
public readonly int InvalidatedSequence;
public readonly Texture Texture;
@ -24,6 +24,17 @@ namespace Ryujinx.Graphics.Gpu.Image
InvalidatedSequence = texture.InvalidatedSequence;
Texture = texture;
}
/// <summary>
/// Create a new entry on the short duration texture cache from the auto delete cache.
/// </summary>
/// <param name="texture">The texture</param>
public ShortTextureCacheEntry(Texture texture)
{
IsAutoDelete = true;
InvalidatedSequence = texture.InvalidatedSequence;
Texture = texture;
}
}
/// <summary>
@ -199,7 +210,11 @@ namespace Ryujinx.Graphics.Gpu.Image
{
texture.DecrementReferenceCount();
_shortCacheLookup.Remove(texture.ShortCacheEntry.Descriptor);
if (!texture.ShortCacheEntry.IsAutoDelete)
{
_shortCacheLookup.Remove(texture.ShortCacheEntry.Descriptor);
}
texture.ShortCacheEntry = null;
}
}
@ -222,6 +237,25 @@ namespace Ryujinx.Graphics.Gpu.Image
texture.IncrementReferenceCount();
}
/// <summary>
/// Adds a texture to the short duration cache without a descriptor. This typically keeps it alive for two ticks.
/// On expiry, it will be removed from the AutoDeleteCache.
/// </summary>
/// <param name="texture">Texture to add to the short cache</param>
public void AddShortCache(Texture texture)
{
if (texture.ShortCacheEntry != null)
{
var entry = new ShortTextureCacheEntry(texture);
_shortCacheBuilder.Add(entry);
texture.ShortCacheEntry = entry;
texture.IncrementReferenceCount();
}
}
/// <summary>
/// Delete textures from the short duration cache.
/// Moves the builder set to be deleted on next process.
@ -234,7 +268,15 @@ namespace Ryujinx.Graphics.Gpu.Image
{
entry.Texture.DecrementReferenceCount();
_shortCacheLookup.Remove(entry.Descriptor);
if (entry.IsAutoDelete)
{
Remove(entry.Texture, false);
}
else
{
_shortCacheLookup.Remove(entry.Descriptor);
}
entry.Texture.ShortCacheEntry = null;
}

View File

@ -144,6 +144,11 @@ namespace Ryujinx.Graphics.Gpu.Image
/// </summary>
public ShortTextureCacheEntry ShortCacheEntry { get; set; }
/// <summary>
/// Whether this texture has ever been referenced by a pool.
/// </summary>
public bool HadPoolOwner { get; private set; }
/// Physical memory ranges where the texture data is located.
/// </summary>
public MultiRange Range { get; private set; }
@ -1423,7 +1428,7 @@ namespace Ryujinx.Graphics.Gpu.Image
_scaledSetScore = Math.Max(0, _scaledSetScore - 1);
}
if (_modifiedStale || Group.HasCopyDependencies)
if (_modifiedStale || Group.HasCopyDependencies || Group.HasFlushBuffer)
{
_modifiedStale = false;
Group.SignalModifying(this, bound);
@ -1506,10 +1511,13 @@ namespace Ryujinx.Graphics.Gpu.Image
/// <param name="gpuVa">GPU VA of the pool reference</param>
public void IncrementReferenceCount(TexturePool pool, int id, ulong gpuVa)
{
HadPoolOwner = true;
lock (_poolOwners)
{
_poolOwners.Add(new TexturePoolOwner { Pool = pool, ID = id, GpuAddress = gpuVa });
}
_referenceCount++;
if (ShortCacheEntry != null)
@ -1594,7 +1602,7 @@ namespace Ryujinx.Graphics.Gpu.Image
_poolOwners.Clear();
}
if (ShortCacheEntry != null && _context.IsGpuThread())
if (ShortCacheEntry != null && !ShortCacheEntry.IsAutoDelete && _context.IsGpuThread())
{
// If this is called from another thread (unmapped), the short cache will
// have to remove this texture on a future tick.
@ -1610,6 +1618,8 @@ namespace Ryujinx.Graphics.Gpu.Image
/// </summary>
public void UpdatePoolMappings()
{
ChangedMapping = true;
lock (_poolOwners)
{
ulong address = 0;
@ -1683,10 +1693,10 @@ namespace Ryujinx.Graphics.Gpu.Image
if (Group.Storage == this)
{
Group.Unmapped();
Group.ClearModified(unmapRange);
}
UpdatePoolMappings();
}
/// <summary>

View File

@ -64,7 +64,7 @@ namespace Ryujinx.Graphics.Gpu.Image
}
/// <summary>
/// Handles removal of textures written to a memory region being unmapped.
/// Handles marking of textures written to a memory region being (partially) remapped.
/// </summary>
/// <param name="sender">Sender object</param>
/// <param name="e">Event arguments</param>
@ -80,26 +80,41 @@ namespace Ryujinx.Graphics.Gpu.Image
overlapCount = _textures.FindOverlaps(unmapped, ref overlaps);
}
for (int i = 0; i < overlapCount; i++)
if (overlapCount > 0)
{
overlaps[i].Unmapped(unmapped);
for (int i = 0; i < overlapCount; i++)
{
overlaps[i].Unmapped(unmapped);
}
}
// If any range was previously unmapped, we also need to purge
// all partially mapped texture, as they might be fully mapped now.
for (int i = 0; i < unmapped.Count; i++)
lock (_partiallyMappedTextures)
{
if (unmapped.GetSubRange(i).Address == MemoryManager.PteUnmapped)
if (overlapCount > 0 || _partiallyMappedTextures.Count > 0)
{
lock (_partiallyMappedTextures)
e.AddRemapAction(() =>
{
foreach (var texture in _partiallyMappedTextures)
lock (_partiallyMappedTextures)
{
texture.Unmapped(unmapped);
}
}
if (overlapCount > 0)
{
for (int i = 0; i < overlapCount; i++)
{
_partiallyMappedTextures.Add(overlaps[i]);
}
}
break;
// Any texture that has been unmapped at any point or is partially unmapped
// should update their pool references after the remap completes.
MultiRange unmapped = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
foreach (var texture in _partiallyMappedTextures)
{
texture.UpdatePoolMappings();
}
}
});
}
}
}
@ -833,7 +848,17 @@ namespace Ryujinx.Graphics.Gpu.Image
if (overlapInCache)
{
_cache.Remove(overlap, flush);
if (flush || overlap.HadPoolOwner || overlap.IsView)
{
_cache.Remove(overlap, flush);
}
else
{
// This texture has only ever been referenced in the AutoDeleteCache.
// Keep this texture alive with the short duration cache, as it may be used often but not sampled.
_cache.AddShortCache(overlap);
}
}
removeOverlap = modified;
@ -1135,6 +1160,44 @@ namespace Ryujinx.Graphics.Gpu.Image
}
}
/// <summary>
/// Queries a texture's memory range and marks it as partially mapped or not.
/// Partially mapped textures re-evaluate their memory range after each time GPU memory is mapped.
/// </summary>
/// <param name="memoryManager">GPU memory manager where the texture is mapped</param>
/// <param name="address">The virtual address of the texture</param>
/// <param name="texture">The texture to be marked</param>
/// <returns>The physical regions for the texture, found when evaluating whether the texture was partially mapped</returns>
public MultiRange UpdatePartiallyMapped(MemoryManager memoryManager, ulong address, Texture texture)
{
MultiRange range;
lock (_partiallyMappedTextures)
{
range = memoryManager.GetPhysicalRegions(address, texture.Size);
bool partiallyMapped = false;
for (int i = 0; i < range.Count; i++)
{
if (range.GetSubRange(i).Address == MemoryManager.PteUnmapped)
{
partiallyMapped = true;
break;
}
}
if (partiallyMapped)
{
_partiallyMappedTextures.Add(texture);
}
else
{
_partiallyMappedTextures.Remove(texture);
}
}
return range;
}
/// <summary>
/// Adds a texture to the short duration cache. This typically keeps it alive for two ticks.
/// </summary>
@ -1145,6 +1208,16 @@ namespace Ryujinx.Graphics.Gpu.Image
_cache.AddShortCache(texture, ref descriptor);
}
/// <summary>
/// Adds a texture to the short duration cache without a descriptor. This typically keeps it alive for two ticks.
/// On expiry, it will be removed from the AutoDeleteCache.
/// </summary>
/// <param name="texture">Texture to add to the short cache</param>
public void AddShortCache(Texture texture)
{
_cache.AddShortCache(texture);
}
/// <summary>
/// Removes a texture from the short duration cache.
/// </summary>

View File

@ -57,6 +57,12 @@ namespace Ryujinx.Graphics.Gpu.Image
/// </summary>
public bool HasCopyDependencies { get; set; }
/// <summary>
/// Indicates if the texture group has a pre-emptive flush buffer.
/// When one is present, the group must always be notified on unbind.
/// </summary>
public bool HasFlushBuffer => _flushBuffer != BufferHandle.Null;
/// <summary>
/// Indicates if this texture has any incompatible overlaps alive.
/// </summary>
@ -89,6 +95,10 @@ namespace Ryujinx.Graphics.Gpu.Image
private bool _incompatibleOverlapsDirty = true;
private bool _flushIncompatibleOverlaps;
private BufferHandle _flushBuffer;
private bool _flushBufferImported;
private bool _flushBufferInvalid;
/// <summary>
/// Create a new texture group.
/// </summary>
@ -464,8 +474,9 @@ namespace Ryujinx.Graphics.Gpu.Image
/// </remarks>
/// <param name="tracked">True if writing the texture data is tracked, false otherwise</param>
/// <param name="sliceIndex">The index of the slice to flush</param>
/// <param name="inBuffer">Whether the flushed texture data is up to date in the flush buffer</param>
/// <param name="texture">The specific host texture to flush. Defaults to the storage texture</param>
private void FlushTextureDataSliceToGuest(bool tracked, int sliceIndex, ITexture texture = null)
private void FlushTextureDataSliceToGuest(bool tracked, int sliceIndex, bool inBuffer, ITexture texture = null)
{
(int layer, int level) = GetLayerLevelForView(sliceIndex);
@ -475,7 +486,16 @@ namespace Ryujinx.Graphics.Gpu.Image
using WritableRegion region = _physicalMemory.GetWritableRegion(Storage.Range.Slice((ulong)offset, (ulong)size), tracked);
Storage.GetTextureDataSliceFromGpu(region.Memory.Span, layer, level, tracked, texture);
if (inBuffer)
{
using PinnedSpan<byte> data = _context.Renderer.GetBufferData(_flushBuffer, offset, size);
Storage.ConvertFromHostCompatibleFormat(region.Memory.Span, data.Get(), level, true);
}
else
{
Storage.GetTextureDataSliceFromGpu(region.Memory.Span, layer, level, tracked, texture);
}
}
/// <summary>
@ -484,12 +504,13 @@ namespace Ryujinx.Graphics.Gpu.Image
/// <param name="tracked">True if writing the texture data is tracked, false otherwise</param>
/// <param name="sliceStart">The first slice to flush</param>
/// <param name="sliceEnd">The slice to finish flushing on (exclusive)</param>
/// <param name="inBuffer">Whether the flushed texture data is up to date in the flush buffer</param>
/// <param name="texture">The specific host texture to flush. Defaults to the storage texture</param>
private void FlushSliceRange(bool tracked, int sliceStart, int sliceEnd, ITexture texture = null)
private void FlushSliceRange(bool tracked, int sliceStart, int sliceEnd, bool inBuffer, ITexture texture = null)
{
for (int i = sliceStart; i < sliceEnd; i++)
{
FlushTextureDataSliceToGuest(tracked, i, texture);
FlushTextureDataSliceToGuest(tracked, i, inBuffer, texture);
}
}
@ -520,7 +541,7 @@ namespace Ryujinx.Graphics.Gpu.Image
{
if (endSlice > startSlice)
{
FlushSliceRange(tracked, startSlice, endSlice);
FlushSliceRange(tracked, startSlice, endSlice, false);
flushed = true;
}
@ -553,7 +574,7 @@ namespace Ryujinx.Graphics.Gpu.Image
}
else
{
FlushSliceRange(tracked, startSlice, endSlice);
FlushSliceRange(tracked, startSlice, endSlice, false);
}
flushed = true;
@ -565,6 +586,58 @@ namespace Ryujinx.Graphics.Gpu.Image
return flushed;
}
/// <summary>
/// Flush the texture data into a persistently mapped buffer.
/// If the buffer does not exist, this method will create it.
/// </summary>
/// <param name="handle">Handle of the texture group to flush slices of</param>
public void FlushIntoBuffer(TextureGroupHandle handle)
{
// Ensure that the buffer exists.
if (_flushBufferInvalid && _flushBuffer != BufferHandle.Null)
{
_flushBufferInvalid = false;
_context.Renderer.DeleteBuffer(_flushBuffer);
_flushBuffer = BufferHandle.Null;
}
if (_flushBuffer == BufferHandle.Null)
{
if (!TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities))
{
return;
}
bool canImport = Storage.Info.IsLinear && Storage.Info.Stride >= Storage.Info.Width * Storage.Info.FormatInfo.BytesPerPixel;
var hostPointer = canImport ? _physicalMemory.GetHostPointer(Storage.Range) : 0;
if (hostPointer != 0 && _context.Renderer.PrepareHostMapping(hostPointer, Storage.Size))
{
_flushBuffer = _context.Renderer.CreateBuffer(hostPointer, (int)Storage.Size);
_flushBufferImported = true;
}
else
{
_flushBuffer = _context.Renderer.CreateBuffer((int)Storage.Size, BufferAccess.FlushPersistent);
_flushBufferImported = false;
}
Storage.BlacklistScale();
}
int sliceStart = handle.BaseSlice;
int sliceEnd = sliceStart + handle.SliceCount;
for (int i = sliceStart; i < sliceEnd; i++)
{
(int layer, int level) = GetLayerLevelForView(i);
Storage.GetFlushTexture().CopyTo(new BufferRange(_flushBuffer, _allOffsets[i], _sliceSizes[level]), layer, level, _flushBufferImported ? Storage.Info.Stride : 0);
}
}
/// <summary>
/// Clears competing modified flags for all incompatible ranges, if they have possibly been modified.
/// </summary>
@ -1570,10 +1643,7 @@ namespace Ryujinx.Graphics.Gpu.Image
_context.Renderer.BackgroundContextAction(() =>
{
if (!isGpuThread)
{
handle.Sync(_context);
}
bool inBuffer = !isGpuThread && handle.Sync(_context);
Storage.SignalModifiedDirty();
@ -1585,13 +1655,24 @@ namespace Ryujinx.Graphics.Gpu.Image
}
}
if (TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities))
if (TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities) && !(inBuffer && _flushBufferImported))
{
FlushSliceRange(false, handle.BaseSlice, handle.BaseSlice + handle.SliceCount, Storage.GetFlushTexture());
FlushSliceRange(false, handle.BaseSlice, handle.BaseSlice + handle.SliceCount, inBuffer, Storage.GetFlushTexture());
}
});
}
/// <summary>
/// Called if any part of the storage texture is unmapped.
/// </summary>
public void Unmapped()
{
if (_flushBufferImported)
{
_flushBufferInvalid = true;
}
}
/// <summary>
/// Dispose this texture group, disposing all related memory tracking handles.
/// </summary>
@ -1606,6 +1687,11 @@ namespace Ryujinx.Graphics.Gpu.Image
{
incompatible.Group._incompatibleOverlaps.RemoveAll(overlap => overlap.Group == this);
}
if (_flushBuffer != BufferHandle.Null)
{
_context.Renderer.DeleteBuffer(_flushBuffer);
}
}
}
}

View File

@ -1,4 +1,5 @@
using Ryujinx.Cpu.Tracking;
using Ryujinx.Graphics.Gpu.Synchronization;
using System;
using System.Collections.Generic;
using System.Linq;
@ -13,8 +14,14 @@ namespace Ryujinx.Graphics.Gpu.Image
/// Also tracks copy dependencies for the handle - references to other handles that must be kept
/// in sync with this one before use.
/// </summary>
class TextureGroupHandle : IDisposable
class TextureGroupHandle : ISyncActionHandler, IDisposable
{
private const int FlushBalanceIncrement = 6;
private const int FlushBalanceWriteCost = 1;
private const int FlushBalanceThreshold = 7;
private const int FlushBalanceMax = 60;
private const int FlushBalanceMin = -10;
private TextureGroup _group;
private int _bindCount;
private int _firstLevel;
@ -26,6 +33,8 @@ namespace Ryujinx.Graphics.Gpu.Image
/// The sync number last registered.
/// </summary>
private ulong _registeredSync;
private ulong _registeredBufferSync = ulong.MaxValue;
private ulong _registeredBufferGuestSync = ulong.MaxValue;
/// <summary>
/// The sync number when the texture was last modified by GPU.
@ -42,6 +51,12 @@ namespace Ryujinx.Graphics.Gpu.Image
/// </summary>
private bool _syncActionRegistered;
/// <summary>
/// Determines the balance of synced writes to flushes.
/// Used to determine if the texture should always write data to a persistent buffer for flush.
/// </summary>
private int _flushBalance;
/// <summary>
/// The byte offset from the start of the storage of this handle.
/// </summary>
@ -132,6 +147,12 @@ namespace Ryujinx.Graphics.Gpu.Image
}
Handles = handles;
if (group.Storage.Info.IsLinear)
{
// Linear textures are presumed to be used for readback initially.
_flushBalance = FlushBalanceThreshold + FlushBalanceIncrement;
}
}
/// <summary>
@ -159,6 +180,35 @@ namespace Ryujinx.Graphics.Gpu.Image
}
}
/// <summary>
/// Determine if the next sync will copy into the flush buffer.
/// </summary>
/// <returns>True if it will copy, false otherwise</returns>
private bool NextSyncCopies()
{
return _flushBalance - FlushBalanceWriteCost > FlushBalanceThreshold;
}
/// <summary>
/// Alters the flush balance by the given value. Should increase significantly with each sync, decrease with each write.
/// A flush balance higher than the threshold will cause a texture to repeatedly copy to a flush buffer on each use.
/// </summary>
/// <param name="modifier">Value to add to the existing flush balance</param>
/// <returns>True if the new balance is over the threshold, false otherwise</returns>
private bool ModifyFlushBalance(int modifier)
{
int result;
int existingBalance;
do
{
existingBalance = _flushBalance;
result = Math.Max(FlushBalanceMin, Math.Min(FlushBalanceMax, existingBalance + modifier));
}
while (Interlocked.CompareExchange(ref _flushBalance, result, existingBalance) != existingBalance);
return result > FlushBalanceThreshold;
}
/// <summary>
/// Adds a single texture view as an overlap if its range overlaps.
/// </summary>
@ -204,7 +254,7 @@ namespace Ryujinx.Graphics.Gpu.Image
if (!_syncActionRegistered)
{
_modifiedSync = context.SyncNumber;
context.RegisterSyncAction(SyncAction, true);
context.RegisterSyncAction(this, true);
_syncActionRegistered = true;
}
@ -241,6 +291,13 @@ namespace Ryujinx.Graphics.Gpu.Image
{
SignalModified(context);
if (!bound && _syncActionRegistered && NextSyncCopies())
{
// On unbind, textures that flush often should immediately create sync so their result can be obtained as soon as possible.
context.CreateHostSyncIfNeeded(HostSyncFlags.Force);
}
// Note: Bind count currently resets to 0 on inherit for safety, as the handle <-> view relationship can change.
_bindCount = Math.Max(0, _bindCount + (bound ? 1 : -1));
}
@ -266,25 +323,35 @@ namespace Ryujinx.Graphics.Gpu.Image
/// removing the modified flag if it was reached, or leaving it set if it has not yet been created.
/// </summary>
/// <param name="context">The GPU context used to wait for sync</param>
public void Sync(GpuContext context)
/// <returns>True if the texture data can be read from the flush buffer</returns>
public bool Sync(GpuContext context)
{
ulong registeredSync = _registeredSync;
long diff = (long)(context.SyncNumber - registeredSync);
// Currently assumes the calling thread is a guest thread.
bool inBuffer = _registeredBufferGuestSync != ulong.MaxValue;
ulong sync = inBuffer ? _registeredBufferGuestSync : _registeredSync;
long diff = (long)(context.SyncNumber - sync);
ModifyFlushBalance(FlushBalanceIncrement);
if (diff > 0)
{
context.Renderer.WaitSync(registeredSync);
context.Renderer.WaitSync(sync);
if ((long)(_modifiedSync - registeredSync) > 0)
if ((long)(_modifiedSync - sync) > 0)
{
// Flush the data in a previous state. Do not remove the modified flag - it will be removed to ignore following writes.
return;
return inBuffer;
}
Modified = false;
return inBuffer;
}
// If the difference is <= 0, no data is not ready yet. Flush any data we can without waiting or removing modified flag.
return false;
}
/// <summary>
@ -296,15 +363,41 @@ namespace Ryujinx.Graphics.Gpu.Image
Interlocked.Exchange(ref _actionRegistered, 0);
}
/// <summary>
/// Action to perform before a sync number is registered after modification.
/// This action will copy the texture data to the flush buffer if this texture
/// flushes often enough, which is determined by the flush balance.
/// </summary>
/// <inheritdoc/>
public void SyncPreAction(bool syncpoint)
{
if (syncpoint || NextSyncCopies())
{
if (ModifyFlushBalance(0) && _registeredBufferSync != _modifiedSync)
{
_group.FlushIntoBuffer(this);
_registeredBufferSync = _modifiedSync;
}
}
}
/// <summary>
/// Action to perform when a sync number is registered after modification.
/// This action will register a read tracking action on the memory tracking handle so that a flush from CPU can happen.
/// </summary>
private void SyncAction()
/// <inheritdoc/>
public bool SyncAction(bool syncpoint)
{
// The storage will need to signal modified again to update the sync number in future.
_group.Storage.SignalModifiedDirty();
bool lastInBuffer = _registeredBufferSync == _modifiedSync;
if (!lastInBuffer)
{
_registeredBufferSync = ulong.MaxValue;
}
lock (Overlaps)
{
foreach (Texture texture in Overlaps)
@ -314,6 +407,7 @@ namespace Ryujinx.Graphics.Gpu.Image
}
// Register region tracking for CPU? (again)
_registeredSync = _modifiedSync;
_syncActionRegistered = false;
@ -321,6 +415,14 @@ namespace Ryujinx.Graphics.Gpu.Image
{
_group.RegisterAction(this);
}
if (syncpoint)
{
_registeredBufferGuestSync = _registeredBufferSync;
}
// If the last modification is in the buffer, keep this sync action alive until it sees a syncpoint.
return syncpoint || !lastInBuffer;
}
/// <summary>

View File

@ -272,7 +272,15 @@ namespace Ryujinx.Graphics.Gpu.Image
ulong address = descriptor.UnpackAddress();
MultiRange range = _channel.MemoryManager.GetPhysicalRegions(address, texture.Size);
if (!descriptor.Equals(ref DescriptorCache[request.ID]))
{
// If the pool entry has already been replaced, just remove the texture.
texture.DecrementReferenceCount();
continue;
}
MultiRange range = _channel.MemoryManager.Physical.TextureCache.UpdatePartiallyMapped(_channel.MemoryManager, address, texture);
// If the texture is not mapped at all, delete its reference.

View File

@ -1,5 +1,6 @@
using Ryujinx.Cpu.Tracking;
using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Synchronization;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
using System;
@ -11,7 +12,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <summary>
/// Buffer, used to store vertex and index data, uniform and storage buffers, and others.
/// </summary>
class Buffer : IRange, IDisposable
class Buffer : IRange, ISyncActionHandler, IDisposable
{
private const ulong GranularBufferThreshold = 4096;
@ -248,7 +249,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (!_syncActionRegistered)
{
_context.RegisterSyncAction(SyncAction);
_context.RegisterSyncAction(this);
_syncActionRegistered = true;
}
}
@ -267,7 +268,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// Action to be performed when a syncpoint is reached after modification.
/// This will register read/write tracking to flush the buffer from GPU when its memory is used.
/// </summary>
private void SyncAction()
/// <inheritdoc/>
public bool SyncAction(bool syncpoint)
{
_syncActionRegistered = false;
@ -284,6 +286,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
_memoryTracking.RegisterAction(_externalFlushDelegate);
SynchronizeMemory(Address, Size);
}
return true;
}
/// <summary>
@ -296,7 +300,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
if (from._syncActionRegistered && !_syncActionRegistered)
{
_context.RegisterSyncAction(SyncAction);
_context.RegisterSyncAction(this);
_syncActionRegistered = true;
}

View File

@ -365,6 +365,22 @@ namespace Ryujinx.Graphics.Gpu.Memory
}
}
/// <summary>
/// Runs remap actions that are added to an unmap event.
/// These must run after the mapping completes.
/// </summary>
/// <param name="e">Event with remap actions</param>
private void RunRemapActions(UnmapEventArgs e)
{
if (e.RemapActions != null)
{
foreach (Action action in e.RemapActions)
{
action();
}
}
}
/// <summary>
/// Maps a given range of pages to the specified CPU virtual address.
/// </summary>
@ -379,12 +395,15 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
lock (_pageTable)
{
MemoryUnmapped?.Invoke(this, new UnmapEventArgs(va, size));
UnmapEventArgs e = new(va, size);
MemoryUnmapped?.Invoke(this, e);
for (ulong offset = 0; offset < size; offset += PageSize)
{
SetPte(va + offset, PackPte(pa + offset, kind));
}
RunRemapActions(e);
}
}
@ -398,12 +417,15 @@ namespace Ryujinx.Graphics.Gpu.Memory
lock (_pageTable)
{
// Event handlers are not expected to be thread safe.
MemoryUnmapped?.Invoke(this, new UnmapEventArgs(va, size));
UnmapEventArgs e = new(va, size);
MemoryUnmapped?.Invoke(this, e);
for (ulong offset = 0; offset < size; offset += PageSize)
{
SetPte(va + offset, PteUnmapped);
}
RunRemapActions(e);
}
}

View File

@ -8,6 +8,7 @@ using Ryujinx.Memory.Tracking;
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Linq;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory
@ -82,6 +83,34 @@ namespace Ryujinx.Graphics.Gpu.Memory
}
}
/// <summary>
/// Gets a host pointer for a given range of application memory.
/// If the memory region is not a single contiguous block, this method returns 0.
/// </summary>
/// <remarks>
/// Getting a host pointer is unsafe. It should be considered invalid immediately if the GPU memory is unmapped.
/// </remarks>
/// <param name="range">Ranges of physical memory where the target data is located</param>
/// <returns>Pointer to the range of memory</returns>
public nint GetHostPointer(MultiRange range)
{
if (range.Count == 1)
{
var singleRange = range.GetSubRange(0);
if (singleRange.Address != MemoryManager.PteUnmapped)
{
var regions = _cpuMemory.GetHostRegions(singleRange.Address, singleRange.Size);
if (regions != null && regions.Count() == 1)
{
return (nint)regions.First().Address;
}
}
}
return 0;
}
/// <summary>
/// Gets a span of data from the application process.
/// </summary>

View File

@ -1,14 +1,24 @@
namespace Ryujinx.Graphics.Gpu.Memory
using System;
using System.Collections.Generic;
namespace Ryujinx.Graphics.Gpu.Memory
{
public class UnmapEventArgs
{
public ulong Address { get; }
public ulong Size { get; }
public List<Action> RemapActions { get; private set; }
public UnmapEventArgs(ulong address, ulong size)
{
Address = address;
Size = size;
}
public void AddRemapAction(Action action)
{
RemapActions ??= new List<Action>();
RemapActions.Add(action);
}
}
}

View File

@ -0,0 +1,30 @@
using System;
namespace Ryujinx.Graphics.Gpu.Synchronization
{
/// <summary>
/// Modifier flags for creating host sync.
/// </summary>
[Flags]
internal enum HostSyncFlags
{
None = 0,
/// <summary>
/// Present if host sync is being created by a syncpoint.
/// </summary>
Syncpoint = 1 << 0,
/// <summary>
/// Present if the sync should signal as soon as possible.
/// </summary>
Strict = 1 << 1,
/// <summary>
/// Present will force the sync to be created, even if no actions are eligible.
/// </summary>
Force = 1 << 2,
StrictSyncpoint = Strict | Syncpoint
}
}

View File

@ -0,0 +1,22 @@
namespace Ryujinx.Graphics.Gpu.Synchronization
{
/// <summary>
/// This interface indicates that a class can be registered for a sync action.
/// </summary>
interface ISyncActionHandler
{
/// <summary>
/// Action to be performed when some synchronizing action is reached after modification.
/// Generally used to register read/write tracking to flush resources from GPU when their memory is used.
/// </summary>
/// <param name="syncpoint">True if the action is a guest syncpoint</param>
/// <returns>True if the action is to be removed, false otherwise</returns>
bool SyncAction(bool syncpoint);
/// <summary>
/// Action to be performed immediately before sync is created.
/// </summary>
/// <param name="syncpoint">True if the action is a guest syncpoint</param>
void SyncPreAction(bool syncpoint) { }
}
}

View File

@ -42,6 +42,20 @@ namespace Ryujinx.Graphics.OpenGL
return Handle.FromInt32<BufferHandle>(handle);
}
public static BufferHandle CreatePersistent(int size)
{
int handle = GL.GenBuffer();
GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle);
GL.BufferStorage(BufferTarget.CopyWriteBuffer, size, IntPtr.Zero,
BufferStorageFlags.MapPersistentBit |
BufferStorageFlags.MapCoherentBit |
BufferStorageFlags.ClientStorageBit |
BufferStorageFlags.MapReadBit);
return Handle.FromInt32<BufferHandle>(handle);
}
public static void Copy(BufferHandle source, BufferHandle destination, int srcOffset, int dstOffset, int size)
{
GL.BindBuffer(BufferTarget.CopyReadBuffer, source.ToInt32());
@ -60,7 +74,11 @@ namespace Ryujinx.Graphics.OpenGL
// Data in the persistent buffer and host array is guaranteed to be available
// until the next time the host thread requests data.
if (HwCapabilities.UsePersistentBufferForFlush)
if (renderer.PersistentBuffers.TryGet(buffer, out IntPtr ptr))
{
return new PinnedSpan<byte>(IntPtr.Add(ptr, offset).ToPointer(), size);
}
else if (HwCapabilities.UsePersistentBufferForFlush)
{
return PinnedSpan<byte>.UnsafeFromSpan(renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size));
}

View File

@ -49,6 +49,11 @@ namespace Ryujinx.Graphics.OpenGL.Image
return GetData();
}
public void CopyTo(BufferRange range, int layer, int level, int stride)
{
throw new NotImplementedException();
}
public void SetData(SpanOrArray<byte> data)
{
var dataSpan = data.AsSpan();

View File

@ -3,6 +3,7 @@ using Ryujinx.Common;
using Ryujinx.Common.Memory;
using Ryujinx.Graphics.GAL;
using System;
using System.Diagnostics;
namespace Ryujinx.Graphics.OpenGL.Image
{
@ -287,6 +288,26 @@ namespace Ryujinx.Graphics.OpenGL.Image
}
}
public void CopyTo(BufferRange range, int layer, int level, int stride)
{
if (stride != 0 && stride != BitUtils.AlignUp(Info.Width * Info.BytesPerPixel, 4))
{
throw new NotSupportedException("Stride conversion for texture copy to buffer not supported.");
}
GL.BindBuffer(BufferTarget.PixelPackBuffer, range.Handle.ToInt32());
FormatInfo format = FormatTable.GetFormatInfo(Info.Format);
if (format.PixelFormat == PixelFormat.DepthStencil)
{
throw new InvalidOperationException("DepthStencil copy to buffer is not supported for layer/level > 0.");
}
int offset = WriteToPbo2D(range.Offset, layer, level);
Debug.Assert(offset == 0);
}
public void WriteToPbo(int offset, bool forceBgra)
{
WriteTo(IntPtr.Zero + offset, forceBgra);

View File

@ -58,10 +58,31 @@ namespace Ryujinx.Graphics.OpenGL
}
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{
return CreateBuffer(size, GAL.BufferAccess.Default);
}
public BufferHandle CreateBuffer(int size, GAL.BufferAccess access)
{
BufferCount++;
return Buffer.Create(size);
if (access == GAL.BufferAccess.FlushPersistent)
{
BufferHandle handle = Buffer.CreatePersistent(size);
PersistentBuffers.Map(handle, size);
return handle;
}
else
{
return Buffer.Create(size);
}
}
public BufferHandle CreateBuffer(nint pointer, int size)
{
throw new NotSupportedException();
}
public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info)
@ -88,6 +109,8 @@ namespace Ryujinx.Graphics.OpenGL
public void DeleteBuffer(BufferHandle buffer)
{
PersistentBuffers.Unmap(buffer);
Buffer.Delete(buffer);
}
@ -272,5 +295,10 @@ namespace Ryujinx.Graphics.OpenGL
{
ScreenCaptured?.Invoke(this, bitmap);
}
public bool PrepareHostMapping(nint address, ulong size)
{
return false;
}
}
}

View File

@ -1,8 +1,9 @@
using OpenTK.Graphics.OpenGL;
using OpenTK.Graphics.OpenGL;
using Ryujinx.Common.Logging;
using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.OpenGL.Image;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
@ -13,6 +14,8 @@ namespace Ryujinx.Graphics.OpenGL
private PersistentBuffer _main = new PersistentBuffer();
private PersistentBuffer _background = new PersistentBuffer();
private Dictionary<BufferHandle, IntPtr> _maps = new Dictionary<BufferHandle, IntPtr>();
public PersistentBuffer Default => BackgroundContextWorker.InBackground ? _background : _main;
public void Dispose()
@ -20,6 +23,30 @@ namespace Ryujinx.Graphics.OpenGL
_main?.Dispose();
_background?.Dispose();
}
public void Map(BufferHandle handle, int size)
{
GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle.ToInt32());
IntPtr ptr = GL.MapBufferRange(BufferTarget.CopyWriteBuffer, IntPtr.Zero, size, BufferAccessMask.MapReadBit | BufferAccessMask.MapPersistentBit);
_maps[handle] = ptr;
}
public void Unmap(BufferHandle handle)
{
if (_maps.ContainsKey(handle))
{
GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle.ToInt32());
GL.UnmapBuffer(BufferTarget.CopyWriteBuffer);
_maps.Remove(handle);
}
}
public bool TryGet(BufferHandle handle, out IntPtr ptr)
{
return _maps.TryGetValue(handle, out ptr);
}
}
class PersistentBuffer : IDisposable

View File

@ -33,6 +33,7 @@ namespace Ryujinx.Graphics.Vulkan
private MemoryAllocation _allocation;
private Auto<DisposableBuffer> _buffer;
private Auto<MemoryAllocation> _allocationAuto;
private bool _allocationImported;
private ulong _bufferHandle;
private CacheByRange<BufferHolder> _cachedConvertedBuffers;
@ -81,6 +82,26 @@ namespace Ryujinx.Graphics.Vulkan
_flushLock = new ReaderWriterLock();
}
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, Auto<MemoryAllocation> allocation, int size, BufferAllocationType type, BufferAllocationType currentType, int offset)
{
_gd = gd;
_device = device;
_allocation = allocation.GetUnsafe();
_allocationAuto = allocation;
_allocationImported = true;
_waitable = new MultiFenceHolder(size);
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto);
_bufferHandle = buffer.Handle;
Size = size;
_map = _allocation.HostPointer + offset;
_baseType = type;
_currentType = currentType;
DesiredType = currentType;
_flushLock = new ReaderWriterLock();
}
public bool TryBackingSwap(ref CommandBufferScoped? cbs)
{
if (_swapQueued && DesiredType != _currentType)
@ -775,8 +796,15 @@ namespace Ryujinx.Graphics.Vulkan
_gd.PipelineInternal?.FlushCommandsIfWeightExceeding(_buffer, (ulong)Size);
_buffer.Dispose();
_allocationAuto.Dispose();
_cachedConvertedBuffers.Dispose();
if (_allocationImported)
{
_allocationAuto.DecrementReferenceCount();
}
else
{
_allocationAuto.Dispose();
}
_flushLock.AcquireWriterLock(Timeout.Infinite);

View File

@ -10,7 +10,7 @@ namespace Ryujinx.Graphics.Vulkan
{
class BufferManager : IDisposable
{
private const MemoryPropertyFlags DefaultBufferMemoryFlags =
public const MemoryPropertyFlags DefaultBufferMemoryFlags =
MemoryPropertyFlags.HostVisibleBit |
MemoryPropertyFlags.HostCoherentBit |
MemoryPropertyFlags.HostCachedBit;
@ -40,6 +40,10 @@ namespace Ryujinx.Graphics.Vulkan
BufferUsageFlags.VertexBufferBit |
BufferUsageFlags.TransformFeedbackBufferBitExt;
private const BufferUsageFlags HostImportedBufferUsageFlags =
BufferUsageFlags.TransferSrcBit |
BufferUsageFlags.TransferDstBit;
private readonly Device _device;
private readonly IdList<BufferHolder> _buffers;
@ -48,11 +52,47 @@ namespace Ryujinx.Graphics.Vulkan
public StagingBuffer StagingBuffer { get; }
public MemoryRequirements HostImportedBufferMemoryRequirements { get; }
public BufferManager(VulkanRenderer gd, Device device)
{
_device = device;
_buffers = new IdList<BufferHolder>();
StagingBuffer = new StagingBuffer(gd, this);
HostImportedBufferMemoryRequirements = GetHostImportedUsageRequirements(gd);
}
public unsafe BufferHandle CreateHostImported(VulkanRenderer gd, nint pointer, int size)
{
var usage = HostImportedBufferUsageFlags;
if (gd.Capabilities.SupportsIndirectParameters)
{
usage |= BufferUsageFlags.IndirectBufferBit;
}
var bufferCreateInfo = new BufferCreateInfo()
{
SType = StructureType.BufferCreateInfo,
Size = (ulong)size,
Usage = usage,
SharingMode = SharingMode.Exclusive
};
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
(Auto<MemoryAllocation> allocation, ulong offset) = gd.HostMemoryAllocator.GetExistingAllocation(pointer, (ulong)size);
gd.Api.BindBufferMemory(_device, buffer, allocation.GetUnsafe().Memory, allocation.GetUnsafe().Offset + offset);
var holder = new BufferHolder(gd, _device, buffer, allocation, size, BufferAllocationType.HostMapped, BufferAllocationType.HostMapped, (int)offset);
BufferCount++;
ulong handle64 = (uint)_buffers.Add(holder);
return Unsafe.As<ulong, BufferHandle>(ref handle64);
}
public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default)
@ -75,6 +115,32 @@ namespace Ryujinx.Graphics.Vulkan
return Unsafe.As<ulong, BufferHandle>(ref handle64);
}
public unsafe MemoryRequirements GetHostImportedUsageRequirements(VulkanRenderer gd)
{
var usage = HostImportedBufferUsageFlags;
if (gd.Capabilities.SupportsIndirectParameters)
{
usage |= BufferUsageFlags.IndirectBufferBit;
}
var bufferCreateInfo = new BufferCreateInfo()
{
SType = StructureType.BufferCreateInfo,
Size = (ulong)Environment.SystemPageSize,
Usage = usage,
SharingMode = SharingMode.Exclusive
};
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
gd.Api.DestroyBuffer(_device, buffer, null);
return requirements;
}
public unsafe (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) CreateBacking(
VulkanRenderer gd,
int size,

View File

@ -364,6 +364,15 @@ namespace Ryujinx.Graphics.Vulkan
};
}
public static BufferAllocationType Convert(this BufferAccess access)
{
return access switch
{
BufferAccess.FlushPersistent => BufferAllocationType.HostMapped,
_ => BufferAllocationType.Auto
};
}
private static T2 LogInvalidAndReturn<T1, T2>(T1 value, string name, T2 defaultValue = default)
{
Logger.Debug?.Print(LogClass.Gpu, $"Invalid {name} enum value: {value}.");

View File

@ -41,6 +41,7 @@ namespace Ryujinx.Graphics.Vulkan
public readonly bool SupportsPipelineStatisticsQuery;
public readonly bool SupportsGeometryShader;
public readonly bool SupportsViewportArray2;
public readonly bool SupportsHostImportedMemory;
public readonly uint MinSubgroupSize;
public readonly uint MaxSubgroupSize;
public readonly ShaderStageFlags RequiredSubgroupSizeStages;
@ -75,6 +76,7 @@ namespace Ryujinx.Graphics.Vulkan
bool supportsPipelineStatisticsQuery,
bool supportsGeometryShader,
bool supportsViewportArray2,
bool supportsHostImportedMemory,
uint minSubgroupSize,
uint maxSubgroupSize,
ShaderStageFlags requiredSubgroupSizeStages,
@ -108,6 +110,7 @@ namespace Ryujinx.Graphics.Vulkan
SupportsPipelineStatisticsQuery = supportsPipelineStatisticsQuery;
SupportsGeometryShader = supportsGeometryShader;
SupportsViewportArray2 = supportsViewportArray2;
SupportsHostImportedMemory = supportsHostImportedMemory;
MinSubgroupSize = minSubgroupSize;
MaxSubgroupSize = maxSubgroupSize;
RequiredSubgroupSizeStages = requiredSubgroupSizeStages;

View File

@ -31,6 +31,7 @@ namespace Ryujinx.Graphics.Vulkan
private readonly IProgram _programColorClearSI;
private readonly IProgram _programColorClearUI;
private readonly IProgram _programStrideChange;
private readonly IProgram _programConvertD32S8ToD24S8;
private readonly IProgram _programConvertIndexBuffer;
private readonly IProgram _programConvertIndirectData;
private readonly IProgram _programColorCopyShortening;
@ -158,6 +159,17 @@ namespace Ryujinx.Graphics.Vulkan
new ShaderSource(ShaderBinaries.ColorDrawToMsFragmentShaderSource, colorDrawToMsFragmentBindings, ShaderStage.Fragment, TargetLanguage.Spirv),
});
var convertD32S8ToD24S8Bindings = new ShaderBindings(
new[] { 0 },
new[] { 1, 2 },
Array.Empty<int>(),
Array.Empty<int>());
_programConvertD32S8ToD24S8 = gd.CreateProgramWithMinimalLayout(new[]
{
new ShaderSource(ShaderBinaries.ConvertD32S8ToD24S8ShaderSource, convertD32S8ToD24S8Bindings, ShaderStage.Compute, TargetLanguage.Spirv),
});
var convertIndexBufferBindings = new ShaderBindings(
new[] { 0 },
new[] { 1, 2 },
@ -1644,6 +1656,82 @@ namespace Ryujinx.Graphics.Vulkan
_pipeline.Finish(gd, cbs);
}
public unsafe void ConvertD32S8ToD24S8(VulkanRenderer gd, CommandBufferScoped cbs, BufferHolder src, Auto<DisposableBuffer> dstBufferAuto, int pixelCount, int dstOffset)
{
int inSize = pixelCount * 2 * sizeof(int);
int outSize = pixelCount * sizeof(int);
var srcBufferAuto = src.GetBuffer();
var srcBuffer = srcBufferAuto.Get(cbs, 0, inSize).Value;
var dstBuffer = dstBufferAuto.Get(cbs, dstOffset, outSize).Value;
var access = AccessFlags.ShaderWriteBit;
var stage = PipelineStageFlags.ComputeShaderBit;
BufferHolder.InsertBufferBarrier(
gd,
cbs.CommandBuffer,
srcBuffer,
BufferHolder.DefaultAccessFlags,
AccessFlags.ShaderReadBit,
PipelineStageFlags.AllCommandsBit,
stage,
0,
outSize);
BufferHolder.InsertBufferBarrier(
gd,
cbs.CommandBuffer,
dstBuffer,
BufferHolder.DefaultAccessFlags,
access,
PipelineStageFlags.AllCommandsBit,
stage,
0,
outSize);
const int ParamsBufferSize = sizeof(int) * 2;
Span<int> shaderParams = stackalloc int[2];
shaderParams[0] = pixelCount;
shaderParams[1] = dstOffset;
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
_pipeline.SetCommandBuffer(cbs);
_pipeline.SetUniformBuffers(stackalloc[] { new BufferAssignment(0, new BufferRange(bufferHandle, 0, ParamsBufferSize)) });
Span<Auto<DisposableBuffer>> sbRanges = new Auto<DisposableBuffer>[2];
sbRanges[0] = srcBufferAuto;
sbRanges[1] = dstBufferAuto;
_pipeline.SetStorageBuffers(1, sbRanges);
_pipeline.SetProgram(_programConvertD32S8ToD24S8);
_pipeline.DispatchCompute(1, 1, 1);
gd.BufferManager.Delete(bufferHandle);
_pipeline.Finish(gd, cbs);
BufferHolder.InsertBufferBarrier(
gd,
cbs.CommandBuffer,
dstBuffer,
access,
BufferHolder.DefaultAccessFlags,
stage,
PipelineStageFlags.AllCommandsBit,
0,
outSize);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)

View File

@ -0,0 +1,188 @@
using Ryujinx.Common;
using Ryujinx.Common.Collections;
using Ryujinx.Common.Logging;
using Silk.NET.Vulkan;
using Silk.NET.Vulkan.Extensions.EXT;
using System;
using System.Collections.Generic;
namespace Ryujinx.Graphics.Vulkan
{
internal class HostMemoryAllocator
{
private struct HostMemoryAllocation
{
public readonly Auto<MemoryAllocation> Allocation;
public readonly IntPtr Pointer;
public readonly ulong Size;
public ulong Start => (ulong)Pointer;
public ulong End => (ulong)Pointer + Size;
public HostMemoryAllocation(Auto<MemoryAllocation> allocation, IntPtr pointer, ulong size)
{
Allocation = allocation;
Pointer = pointer;
Size = size;
}
}
private readonly MemoryAllocator _allocator;
private readonly Vk _api;
private readonly ExtExternalMemoryHost _hostMemoryApi;
private readonly Device _device;
private readonly object _lock = new();
private List<HostMemoryAllocation> _allocations;
private IntervalTree<ulong, HostMemoryAllocation> _allocationTree;
public HostMemoryAllocator(MemoryAllocator allocator, Vk api, ExtExternalMemoryHost hostMemoryApi, Device device)
{
_allocator = allocator;
_api = api;
_hostMemoryApi = hostMemoryApi;
_device = device;
_allocations = new List<HostMemoryAllocation>();
_allocationTree = new IntervalTree<ulong, HostMemoryAllocation>();
}
public unsafe bool TryImport(
MemoryRequirements requirements,
MemoryPropertyFlags flags,
IntPtr pointer,
ulong size)
{
lock (_lock)
{
// Does a compatible allocation exist in the tree?
var allocations = new HostMemoryAllocation[10];
ulong start = (ulong)pointer;
ulong end = start + size;
int count = _allocationTree.Get(start, end, ref allocations);
// A compatible range is one that where the start and end completely cover the requested range.
for (int i = 0; i < count; i++)
{
HostMemoryAllocation existing = allocations[i];
if (start >= existing.Start && end <= existing.End)
{
try
{
existing.Allocation.IncrementReferenceCount();
return true;
}
catch (InvalidOperationException)
{
// Can throw if the allocation has been disposed.
// Just continue the search if this happens.
}
}
}
nint pageAlignedPointer = BitUtils.AlignDown(pointer, Environment.SystemPageSize);
nint pageAlignedEnd = BitUtils.AlignUp((nint)((ulong)pointer + size), Environment.SystemPageSize);
ulong pageAlignedSize = (ulong)(pageAlignedEnd - pageAlignedPointer);
Result getResult = _hostMemoryApi.GetMemoryHostPointerProperties(_device, ExternalMemoryHandleTypeFlags.HostAllocationBitExt, (void*)pageAlignedPointer, out MemoryHostPointerPropertiesEXT properties);
if (getResult < Result.Success)
{
return false;
}
int memoryTypeIndex = _allocator.FindSuitableMemoryTypeIndex(properties.MemoryTypeBits & requirements.MemoryTypeBits, flags);
if (memoryTypeIndex < 0)
{
return false;
}
ImportMemoryHostPointerInfoEXT importInfo = new ImportMemoryHostPointerInfoEXT()
{
SType = StructureType.ImportMemoryHostPointerInfoExt,
HandleType = ExternalMemoryHandleTypeFlags.HostAllocationBitExt,
PHostPointer = (void*)pageAlignedPointer
};
var memoryAllocateInfo = new MemoryAllocateInfo()
{
SType = StructureType.MemoryAllocateInfo,
AllocationSize = pageAlignedSize,
MemoryTypeIndex = (uint)memoryTypeIndex,
PNext = &importInfo
};
Result result = _api.AllocateMemory(_device, memoryAllocateInfo, null, out var deviceMemory);
if (result < Result.Success)
{
Logger.Debug?.PrintMsg(LogClass.Gpu, $"Host mapping import 0x{pageAlignedPointer:x16} 0x{pageAlignedSize:x8} failed.");
return false;
}
var allocation = new MemoryAllocation(this, deviceMemory, pageAlignedPointer, 0, pageAlignedSize);
var allocAuto = new Auto<MemoryAllocation>(allocation);
var hostAlloc = new HostMemoryAllocation(allocAuto, pageAlignedPointer, pageAlignedSize);
allocAuto.IncrementReferenceCount();
allocAuto.Dispose(); // Kept alive by ref count only.
// Register this mapping for future use.
_allocationTree.Add(hostAlloc.Start, hostAlloc.End, hostAlloc);
_allocations.Add(hostAlloc);
}
return true;
}
public (Auto<MemoryAllocation>, ulong) GetExistingAllocation(IntPtr pointer, ulong size)
{
lock (_lock)
{
// Does a compatible allocation exist in the tree?
var allocations = new HostMemoryAllocation[10];
ulong start = (ulong)pointer;
ulong end = start + size;
int count = _allocationTree.Get(start, end, ref allocations);
// A compatible range is one that where the start and end completely cover the requested range.
for (int i = 0; i < count; i++)
{
HostMemoryAllocation existing = allocations[i];
if (start >= existing.Start && end <= existing.End)
{
return (existing.Allocation, start - existing.Start);
}
}
throw new InvalidOperationException($"No host allocation was prepared for requested range 0x{pointer:x16}:0x{size:x16}.");
}
}
public void Free(DeviceMemory memory, ulong offset, ulong size)
{
lock (_lock)
{
_allocations.RemoveAll(allocation =>
{
if (allocation.Allocation.GetUnsafe().Memory.Handle == memory.Handle)
{
_allocationTree.Remove(allocation.Start, allocation);
return true;
}
return false;
});
}
_api.FreeMemory(_device, memory, ReadOnlySpan<AllocationCallbacks>.Empty);
}
}
}

View File

@ -7,6 +7,7 @@ namespace Ryujinx.Graphics.Vulkan
{
private readonly MemoryAllocatorBlockList _owner;
private readonly MemoryAllocatorBlockList.Block _block;
private readonly HostMemoryAllocator _hostMemory;
public DeviceMemory Memory { get; }
public IntPtr HostPointer { get;}
@ -29,9 +30,30 @@ namespace Ryujinx.Graphics.Vulkan
Size = size;
}
public MemoryAllocation(
HostMemoryAllocator hostMemory,
DeviceMemory memory,
IntPtr hostPointer,
ulong offset,
ulong size)
{
_hostMemory = hostMemory;
Memory = memory;
HostPointer = hostPointer;
Offset = offset;
Size = size;
}
public void Dispose()
{
_owner.Free(_block, Offset, Size);
if (_hostMemory != null)
{
_hostMemory.Free(Memory, Offset, Size);
}
else
{
_owner.Free(_block, Offset, Size);
}
}
}
}

View File

@ -57,7 +57,7 @@ namespace Ryujinx.Graphics.Vulkan
return newBl.Allocate(size, alignment, map);
}
private int FindSuitableMemoryTypeIndex(
internal int FindSuitableMemoryTypeIndex(
uint memoryTypeBits,
MemoryPropertyFlags flags)
{

View File

@ -0,0 +1,58 @@
#version 450 core
#extension GL_EXT_scalar_block_layout : require
layout (local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
layout (std430, set = 0, binding = 0) uniform stride_arguments
{
int pixelCount;
int dstStartOffset;
};
layout (std430, set = 1, binding = 1) buffer in_s
{
uint[] in_data;
};
layout (std430, set = 1, binding = 2) buffer out_s
{
uint[] out_data;
};
void main()
{
// Determine what slice of the stride copies this invocation will perform.
int invocations = int(gl_WorkGroupSize.x);
int copiesRequired = pixelCount;
// Find the copies that this invocation should perform.
// - Copies that all invocations perform.
int allInvocationCopies = copiesRequired / invocations;
// - Extra remainder copy that this invocation performs.
int index = int(gl_LocalInvocationID.x);
int extra = (index < (copiesRequired % invocations)) ? 1 : 0;
int copyCount = allInvocationCopies + extra;
// Finally, get the starting offset. Make sure to count extra copies.
int startCopy = allInvocationCopies * index + min(copiesRequired % invocations, index);
int srcOffset = startCopy * 2;
int dstOffset = dstStartOffset + startCopy;
// Perform the conversion for this region.
for (int i = 0; i < copyCount; i++)
{
float depth = uintBitsToFloat(in_data[srcOffset++]);
uint stencil = in_data[srcOffset++];
uint rescaledDepth = uint(clamp(depth, 0.0, 1.0) * 16777215.0);
out_data[dstOffset++] = (rescaledDepth << 8) | (stencil & 0xff);
}
}

View File

@ -1236,6 +1236,213 @@ namespace Ryujinx.Graphics.Vulkan.Shaders
0x38, 0x00, 0x01, 0x00,
};
public static readonly byte[] ConvertD32S8ToD24S8ShaderSource = new byte[]
{
0x03, 0x02, 0x23, 0x07, 0x00, 0x05, 0x01, 0x00, 0x0A, 0x00, 0x0D, 0x00, 0x77, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x06, 0x00,
0x01, 0x00, 0x00, 0x00, 0x47, 0x4C, 0x53, 0x4C, 0x2E, 0x73, 0x74, 0x64, 0x2E, 0x34, 0x35, 0x30,
0x00, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x0F, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6D, 0x61, 0x69, 0x6E,
0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x4F, 0x00, 0x00, 0x00,
0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x06, 0x00, 0x04, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00,
0x40, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x03, 0x00,
0x02, 0x00, 0x00, 0x00, 0xC2, 0x01, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x47, 0x4C, 0x5F, 0x45,
0x58, 0x54, 0x5F, 0x73, 0x63, 0x61, 0x6C, 0x61, 0x72, 0x5F, 0x62, 0x6C, 0x6F, 0x63, 0x6B, 0x5F,
0x6C, 0x61, 0x79, 0x6F, 0x75, 0x74, 0x00, 0x00, 0x04, 0x00, 0x0A, 0x00, 0x47, 0x4C, 0x5F, 0x47,
0x4F, 0x4F, 0x47, 0x4C, 0x45, 0x5F, 0x63, 0x70, 0x70, 0x5F, 0x73, 0x74, 0x79, 0x6C, 0x65, 0x5F,
0x6C, 0x69, 0x6E, 0x65, 0x5F, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00, 0x00,
0x04, 0x00, 0x08, 0x00, 0x47, 0x4C, 0x5F, 0x47, 0x4F, 0x4F, 0x47, 0x4C, 0x45, 0x5F, 0x69, 0x6E,
0x63, 0x6C, 0x75, 0x64, 0x65, 0x5F, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00,
0x05, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6D, 0x61, 0x69, 0x6E, 0x00, 0x00, 0x00, 0x00,
0x05, 0x00, 0x05, 0x00, 0x08, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x76, 0x6F, 0x63, 0x61, 0x74, 0x69,
0x6F, 0x6E, 0x73, 0x00, 0x05, 0x00, 0x06, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x63, 0x6F, 0x70, 0x69,
0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x00, 0x00, 0x05, 0x00, 0x07, 0x00,
0x0B, 0x00, 0x00, 0x00, 0x73, 0x74, 0x72, 0x69, 0x64, 0x65, 0x5F, 0x61, 0x72, 0x67, 0x75, 0x6D,
0x65, 0x6E, 0x74, 0x73, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0x0B, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x70, 0x69, 0x78, 0x65, 0x6C, 0x43, 0x6F, 0x75, 0x6E, 0x74, 0x00, 0x00,
0x06, 0x00, 0x07, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x73, 0x74, 0x53,
0x74, 0x61, 0x72, 0x74, 0x4F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00,
0x0D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x07, 0x00, 0x12, 0x00, 0x00, 0x00,
0x61, 0x6C, 0x6C, 0x49, 0x6E, 0x76, 0x6F, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x43, 0x6F, 0x70,
0x69, 0x65, 0x73, 0x00, 0x05, 0x00, 0x04, 0x00, 0x16, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x64, 0x65,
0x78, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x67, 0x6C, 0x5F, 0x4C,
0x6F, 0x63, 0x61, 0x6C, 0x49, 0x6E, 0x76, 0x6F, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x49, 0x44,
0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x20, 0x00, 0x00, 0x00, 0x65, 0x78, 0x74, 0x72,
0x61, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x29, 0x00, 0x00, 0x00, 0x63, 0x6F, 0x70, 0x79,
0x43, 0x6F, 0x75, 0x6E, 0x74, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x2D, 0x00, 0x00, 0x00,
0x73, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6F, 0x70, 0x79, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00,
0x37, 0x00, 0x00, 0x00, 0x73, 0x72, 0x63, 0x4F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x00, 0x00, 0x00,
0x05, 0x00, 0x05, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x64, 0x73, 0x74, 0x4F, 0x66, 0x66, 0x73, 0x65,
0x74, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00,
0x05, 0x00, 0x04, 0x00, 0x4B, 0x00, 0x00, 0x00, 0x64, 0x65, 0x70, 0x74, 0x68, 0x00, 0x00, 0x00,
0x05, 0x00, 0x04, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x5F, 0x73, 0x00, 0x00, 0x00, 0x00,
0x06, 0x00, 0x05, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x5F, 0x64,
0x61, 0x74, 0x61, 0x00, 0x05, 0x00, 0x03, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x05, 0x00, 0x04, 0x00, 0x57, 0x00, 0x00, 0x00, 0x73, 0x74, 0x65, 0x6E, 0x63, 0x69, 0x6C, 0x00,
0x05, 0x00, 0x06, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x72, 0x65, 0x73, 0x63, 0x61, 0x6C, 0x65, 0x64,
0x44, 0x65, 0x70, 0x74, 0x68, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x65, 0x00, 0x00, 0x00,
0x6F, 0x75, 0x74, 0x5F, 0x73, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0x65, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x6F, 0x75, 0x74, 0x5F, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00,
0x05, 0x00, 0x03, 0x00, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00,
0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x48, 0x00, 0x05, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00,
0x04, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x4C, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x48, 0x00, 0x05, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x64, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x48, 0x00, 0x05, 0x00, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x65, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x67, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x67, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x47, 0x00, 0x04, 0x00, 0x76, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00,
0x13, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00, 0x00, 0x21, 0x00, 0x03, 0x00, 0x03, 0x00, 0x00, 0x00,
0x02, 0x00, 0x00, 0x00, 0x15, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00,
0x06, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00,
0x40, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x04, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00,
0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x0B, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00,
0x02, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x06, 0x00, 0x00, 0x00, 0x15, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x04, 0x00, 0x18, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00,
0x03, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x19, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x18, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, 0x19, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x17, 0x00, 0x00, 0x00, 0x14, 0x00, 0x02, 0x00, 0x25, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x16, 0x00, 0x03, 0x00,
0x49, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x4A, 0x00, 0x00, 0x00,
0x07, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x03, 0x00, 0x4C, 0x00, 0x00, 0x00,
0x17, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x03, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x4C, 0x00, 0x00, 0x00,
0x20, 0x00, 0x04, 0x00, 0x4E, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x4D, 0x00, 0x00, 0x00,
0x3B, 0x00, 0x04, 0x00, 0x4E, 0x00, 0x00, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00,
0x20, 0x00, 0x04, 0x00, 0x52, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00,
0x20, 0x00, 0x04, 0x00, 0x56, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00,
0x2B, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x2B, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3F,
0x2B, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x7F, 0x4B,
0x1D, 0x00, 0x03, 0x00, 0x64, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x03, 0x00,
0x65, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x66, 0x00, 0x00, 0x00,
0x0C, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, 0x66, 0x00, 0x00, 0x00,
0x67, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x6B, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00,
0x6E, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00,
0x74, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00,
0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x06, 0x00, 0x18, 0x00, 0x00, 0x00,
0x76, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00,
0x36, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x03, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00, 0x05, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x2D, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x07, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x4A, 0x00, 0x00, 0x00, 0x4B, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x56, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00,
0x56, 0x00, 0x00, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x08, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x0F, 0x00, 0x00, 0x00,
0x10, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x0A, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x13, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x87, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00,
0x15, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x12, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x1C, 0x00, 0x00, 0x00,
0x1D, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x17, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x16, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x21, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x22, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x23, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x8B, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00,
0x24, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0xB1, 0x00, 0x05, 0x00,
0x25, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00,
0xA9, 0x00, 0x06, 0x00, 0x06, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00,
0x27, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, 0x20, 0x00, 0x00, 0x00,
0x28, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2A, 0x00, 0x00, 0x00,
0x12, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00,
0x20, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x00, 0x00,
0x2A, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, 0x29, 0x00, 0x00, 0x00,
0x2C, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2E, 0x00, 0x00, 0x00,
0x12, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2F, 0x00, 0x00, 0x00,
0x16, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00,
0x2E, 0x00, 0x00, 0x00, 0x2F, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x31, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x32, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x8B, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00,
0x33, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x07, 0x00,
0x06, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00,
0x33, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00,
0x36, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x2D, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x38, 0x00, 0x00, 0x00, 0x2D, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00,
0x3A, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x37, 0x00, 0x00, 0x00, 0x3A, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x0F, 0x00, 0x00, 0x00,
0x3C, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x00, 0x00, 0x2D, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00,
0x06, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x00, 0x00,
0x3E, 0x00, 0x03, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x40, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00, 0x41, 0x00, 0x00, 0x00,
0xF8, 0x00, 0x02, 0x00, 0x41, 0x00, 0x00, 0x00, 0xF6, 0x00, 0x04, 0x00, 0x43, 0x00, 0x00, 0x00,
0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00, 0x45, 0x00, 0x00, 0x00,
0xF8, 0x00, 0x02, 0x00, 0x45, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x46, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x47, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0xB1, 0x00, 0x05, 0x00, 0x25, 0x00, 0x00, 0x00,
0x48, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0xFA, 0x00, 0x04, 0x00,
0x48, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00,
0x42, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00,
0x37, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00,
0x50, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, 0x37, 0x00, 0x00, 0x00,
0x51, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x52, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00,
0x4F, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x17, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x04, 0x00,
0x49, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x4B, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,
0x58, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00,
0x59, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00,
0x37, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x52, 0x00, 0x00, 0x00,
0x5A, 0x00, 0x00, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00,
0x3D, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x5B, 0x00, 0x00, 0x00, 0x5A, 0x00, 0x00, 0x00,
0x3E, 0x00, 0x03, 0x00, 0x57, 0x00, 0x00, 0x00, 0x5B, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x49, 0x00, 0x00, 0x00, 0x5D, 0x00, 0x00, 0x00, 0x4B, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x08, 0x00,
0x49, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00,
0x5D, 0x00, 0x00, 0x00, 0x5E, 0x00, 0x00, 0x00, 0x5F, 0x00, 0x00, 0x00, 0x85, 0x00, 0x05, 0x00,
0x49, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00,
0x6D, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00,
0x3E, 0x00, 0x03, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00,
0x06, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00,
0x3E, 0x00, 0x03, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x17, 0x00, 0x00, 0x00, 0x6A, 0x00, 0x00, 0x00, 0x5C, 0x00, 0x00, 0x00, 0xC4, 0x00, 0x05, 0x00,
0x17, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6A, 0x00, 0x00, 0x00, 0x6B, 0x00, 0x00, 0x00,
0x3D, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00,
0xC7, 0x00, 0x05, 0x00, 0x17, 0x00, 0x00, 0x00, 0x6F, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00,
0x6E, 0x00, 0x00, 0x00, 0xC5, 0x00, 0x05, 0x00, 0x17, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00,
0x6C, 0x00, 0x00, 0x00, 0x6F, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x52, 0x00, 0x00, 0x00,
0x71, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00,
0x3E, 0x00, 0x03, 0x00, 0x71, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00,
0x44, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00, 0x44, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00,
0x06, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00,
0x06, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00,
0x3E, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00,
0x41, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00, 0x43, 0x00, 0x00, 0x00, 0xFD, 0x00, 0x01, 0x00,
0x38, 0x00, 0x01, 0x00
};
public static readonly byte[] ConvertIndexBufferShaderSource = new byte[]
{
0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0A, 0x00, 0x08, 0x00, 0x91, 0x00, 0x00, 0x00,

View File

@ -125,6 +125,24 @@ namespace Ryujinx.Graphics.Vulkan
if (result != null)
{
if (result.Waitable == null)
{
return;
}
long beforeTicks = Stopwatch.GetTimestamp();
if (result.NeedsFlush(FlushId))
{
_gd.InterruptAction(() =>
{
if (result.NeedsFlush(FlushId))
{
_gd.FlushAllCommands();
}
});
}
lock (result)
{
if (result.Waitable == null)
@ -132,19 +150,6 @@ namespace Ryujinx.Graphics.Vulkan
return;
}
long beforeTicks = Stopwatch.GetTimestamp();
if (result.NeedsFlush(FlushId))
{
_gd.InterruptAction(() =>
{
if (result.NeedsFlush(FlushId))
{
_gd.FlushAllCommands();
}
});
}
bool signaled = result.Signalled || result.Waitable.WaitForFences(_gd.Api, _device, 1000000000);
if (!signaled)

View File

@ -67,6 +67,11 @@ namespace Ryujinx.Graphics.Vulkan
return GetData();
}
public void CopyTo(BufferRange range, int layer, int level, int stride)
{
throw new NotImplementedException();
}
public void Release()
{
if (_gd.Textures.Remove(this))

View File

@ -563,6 +563,34 @@ namespace Ryujinx.Graphics.Vulkan
}
}
public void CopyTo(BufferRange range, int layer, int level, int stride)
{
_gd.PipelineInternal.EndRenderPass();
var cbs = _gd.PipelineInternal.CurrentCommandBuffer;
int outSize = Info.GetMipSize(level);
int hostSize = GetBufferDataLength(outSize);
var image = GetImage().Get(cbs).Value;
int offset = range.Offset;
Auto<DisposableBuffer> autoBuffer = _gd.BufferManager.GetBuffer(cbs.CommandBuffer, range.Handle, true);
VkBuffer buffer = autoBuffer.Get(cbs, range.Offset, outSize).Value;
if (PrepareOutputBuffer(cbs, hostSize, buffer, out VkBuffer copyToBuffer, out BufferHolder tempCopyHolder))
{
offset = 0;
}
CopyFromOrToBuffer(cbs.CommandBuffer, copyToBuffer, image, hostSize, true, layer, level, 1, 1, singleSlice: true, offset, stride);
if (tempCopyHolder != null)
{
CopyDataToOutputBuffer(cbs, tempCopyHolder, autoBuffer, hostSize, range.Offset);
tempCopyHolder.Dispose();
}
}
private ReadOnlySpan<byte> GetData(CommandBufferPool cbp, PersistentFlushBuffer flushBuffer)
{
int size = 0;
@ -693,6 +721,30 @@ namespace Ryujinx.Graphics.Vulkan
return storage;
}
private bool PrepareOutputBuffer(CommandBufferScoped cbs, int hostSize, VkBuffer target, out VkBuffer copyTarget, out BufferHolder copyTargetHolder)
{
if (NeedsD24S8Conversion())
{
copyTargetHolder = _gd.BufferManager.Create(_gd, hostSize);
copyTarget = copyTargetHolder.GetBuffer().Get(cbs, 0, hostSize).Value;
return true;
}
copyTarget = target;
copyTargetHolder = null;
return false;
}
private void CopyDataToOutputBuffer(CommandBufferScoped cbs, BufferHolder hostData, Auto<DisposableBuffer> copyTarget, int hostSize, int dstOffset)
{
if (NeedsD24S8Conversion())
{
_gd.HelperShader.ConvertD32S8ToD24S8(_gd, cbs, hostData, copyTarget, hostSize / (2 * sizeof(int)), dstOffset);
}
}
private bool NeedsD24S8Conversion()
{
return FormatCapabilities.IsD24S8(Info.Format) && VkFormat == VkFormat.D32SfloatS8Uint;
@ -708,7 +760,9 @@ namespace Ryujinx.Graphics.Vulkan
int dstLevel,
int dstLayers,
int dstLevels,
bool singleSlice)
bool singleSlice,
int offset = 0,
int stride = 0)
{
bool is3D = Info.Target == Target.Texture3D;
int width = Math.Max(1, Info.Width >> dstLevel);
@ -718,8 +772,6 @@ namespace Ryujinx.Graphics.Vulkan
int layers = dstLayers;
int levels = dstLevels;
int offset = 0;
for (int level = 0; level < levels; level++)
{
int mipSize = GetBufferDataLength(Info.GetMipSize2D(dstLevel + level) * dstLayers);
@ -731,7 +783,7 @@ namespace Ryujinx.Graphics.Vulkan
break;
}
int rowLength = (Info.GetMipStride(dstLevel + level) / Info.BytesPerPixel) * Info.BlockWidth;
int rowLength = ((stride == 0 ? Info.GetMipStride(dstLevel + level) : stride) / Info.BytesPerPixel) * Info.BlockWidth;
var aspectFlags = Info.Format.ConvertAspectFlags();

View File

@ -27,6 +27,7 @@ namespace Ryujinx.Graphics.Vulkan
ExtTransformFeedback.ExtensionName,
KhrDrawIndirectCount.ExtensionName,
KhrPushDescriptor.ExtensionName,
ExtExternalMemoryHost.ExtensionName,
"VK_EXT_blend_operation_advanced",
"VK_EXT_custom_border_color",
"VK_EXT_descriptor_indexing", // Enabling this works around an issue with disposed buffer bindings on RADV.

View File

@ -44,6 +44,7 @@ namespace Ryujinx.Graphics.Vulkan
internal object QueueLock { get; private set; }
internal MemoryAllocator MemoryAllocator { get; private set; }
internal HostMemoryAllocator HostMemoryAllocator { get; private set; }
internal CommandBufferPool CommandBufferPool { get; private set; }
internal DescriptorSetManager DescriptorSetManager { get; private set; }
internal PipelineLayoutCache PipelineLayoutCache { get; private set; }
@ -307,6 +308,7 @@ namespace Ryujinx.Graphics.Vulkan
_physicalDevice.PhysicalDeviceFeatures.PipelineStatisticsQuery,
_physicalDevice.PhysicalDeviceFeatures.GeometryShader,
_physicalDevice.IsDeviceExtensionPresent("VK_NV_viewport_array2"),
_physicalDevice.IsDeviceExtensionPresent(ExtExternalMemoryHost.ExtensionName),
propertiesSubgroupSizeControl.MinSubgroupSize,
propertiesSubgroupSizeControl.MaxSubgroupSize,
propertiesSubgroupSizeControl.RequiredSubgroupSizeStages,
@ -319,6 +321,9 @@ namespace Ryujinx.Graphics.Vulkan
MemoryAllocator = new MemoryAllocator(Api, _physicalDevice, _device);
Api.TryGetDeviceExtension(_instance.Instance, _device, out ExtExternalMemoryHost hostMemoryApi);
HostMemoryAllocator = new HostMemoryAllocator(MemoryAllocator, Api, hostMemoryApi, _device);
CommandBufferPool = new CommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex);
DescriptorSetManager = new DescriptorSetManager(_device);
@ -375,11 +380,21 @@ namespace Ryujinx.Graphics.Vulkan
_initialized = true;
}
public BufferHandle CreateBuffer(int size, BufferAccess access)
{
return BufferManager.CreateWithHandle(this, size, access.Convert());
}
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{
return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint);
}
public BufferHandle CreateBuffer(nint pointer, int size)
{
return BufferManager.CreateHostImported(this, pointer, size);
}
public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info)
{
bool isCompute = sources.Length == 1 && sources[0].Stage == ShaderStage.Compute;
@ -816,5 +831,11 @@ namespace Ryujinx.Graphics.Vulkan
// Last step destroy the instance
_instance.Dispose();
}
public bool PrepareHostMapping(nint address, ulong size)
{
return Capabilities.SupportsHostImportedMemory &&
HostMemoryAllocator.TryImport(BufferManager.HostImportedBufferMemoryRequirements, BufferManager.DefaultBufferMemoryFlags, address, size);
}
}
}

View File

@ -57,5 +57,19 @@
return thisAddress < otherEndAddress && otherAddress < thisEndAddress;
}
/// <summary>
/// Returns a string summary of the memory range.
/// </summary>
/// <returns>A string summary of the memory range</returns>
public override string ToString()
{
if (Address == ulong.MaxValue)
{
return $"[Unmapped 0x{Size:X}]";
}
return $"[0x{Address:X}, 0x{EndAddress:X})";
}
}
}

View File

@ -319,5 +319,14 @@ namespace Ryujinx.Memory.Range
return hash.ToHashCode();
}
/// <summary>
/// Returns a string summary of the ranges contained in the MultiRange.
/// </summary>
/// <returns>A string summary of the ranges contained within</returns>
public override string ToString()
{
return HasSingleRange ? _singleRange.ToString() : string.Join(", ", _ranges);
}
}
}