Linux: Automatically increase vm.max_map_count if it's too low (#4702)

* memory: Check results of pinvoke calls

* Increase vm.max_map_count when running Ryujinx

* Add SupportedOSPlatform attribute for WindowsApiException

* Revert increasing vm.max_map_count via script

* Add LinuxHelper to detect and increase vm.max_map_count

With GUI dialogs, this should be a bit more user-friendly.

* Supply arguments as a list to RunPkExec

* Add error logging in case RunPkExec() fails

* Prevent Gtk from crashing

src/Ryujinx.Memory/MemoryManagementUnix.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Concurrent;
+using System.Runtime.InteropServices;
 using System.Runtime.Versioning;
 using System.Text;
 
@@ -53,9 +54,9 @@ namespace Ryujinx.Memory
 
             IntPtr ptr = mmap(IntPtr.Zero, size, prot, flags, -1, 0);
 
-            if (ptr == new IntPtr(-1L))
+            if (ptr == MAP_FAILED)
             {
-                throw new OutOfMemoryException();
+                throw new SystemException(Marshal.GetLastPInvokeErrorMessage());
             }
 
             if (!_allocations.TryAdd(ptr, size))
@@ -76,17 +77,33 @@ namespace Ryujinx.Memory
                 prot |= MmapProts.PROT_EXEC;
             }
 
-            return mprotect(address, size, prot) == 0;
+            if (mprotect(address, size, prot) != 0)
+            {
+                throw new SystemException(Marshal.GetLastPInvokeErrorMessage());
+            }
+
+            return true;
         }
 
         public static bool Decommit(IntPtr address, ulong size)
         {
             // Must be writable for madvise to work properly.
-            mprotect(address, size, MmapProts.PROT_READ | MmapProts.PROT_WRITE);
+            if (mprotect(address, size, MmapProts.PROT_READ | MmapProts.PROT_WRITE) != 0)
+            {
+                throw new SystemException(Marshal.GetLastPInvokeErrorMessage());
+            }
 
-            madvise(address, size, MADV_REMOVE);
+            if (madvise(address, size, MADV_REMOVE) != 0)
+            {
+                throw new SystemException(Marshal.GetLastPInvokeErrorMessage());
+            }
 
-            return mprotect(address, size, MmapProts.PROT_NONE) == 0;
+            if (mprotect(address, size, MmapProts.PROT_NONE) != 0)
+            {
+                throw new SystemException(Marshal.GetLastPInvokeErrorMessage());
+            }
+
+            return true;
         }
 
         public static bool Reprotect(IntPtr address, ulong size, MemoryPermission permission)

src/Ryujinx.Memory/MemoryManagementWindows.cs

@@ -1,5 +1,6 @@
 using Ryujinx.Memory.WindowsShared;
 using System;
+using System.Runtime.InteropServices;
 using System.Runtime.Versioning;
 
 namespace Ryujinx.Memory
@@ -36,7 +37,7 @@ namespace Ryujinx.Memory
 
             if (ptr == IntPtr.Zero)
             {
-                throw new OutOfMemoryException();
+                throw new SystemException(Marshal.GetLastPInvokeErrorMessage());
             }
 
             return ptr;
@@ -48,7 +49,7 @@ namespace Ryujinx.Memory
 
             if (ptr == IntPtr.Zero)
             {
-                throw new OutOfMemoryException();
+                throw new SystemException(Marshal.GetLastPInvokeErrorMessage());
             }
 
             return ptr;

src/Ryujinx.Memory/MemoryManagerUnixHelper.cs

@@ -1,8 +1,11 @@
 using System;
 using System.Runtime.InteropServices;
+using System.Runtime.Versioning;
 
 namespace Ryujinx.Memory
 {
+    [SupportedOSPlatform("linux")]
+    [SupportedOSPlatform("macos")]
     public static partial class MemoryManagerUnixHelper
     {
         [Flags]
@@ -41,6 +44,8 @@ namespace Ryujinx.Memory
             O_SYNC = 256,
         }
 
+        public const IntPtr MAP_FAILED = -1;
+
         private const int MAP_ANONYMOUS_LINUX_GENERIC = 0x20;
         private const int MAP_NORESERVE_LINUX_GENERIC = 0x4000;
         private const int MAP_UNLOCKED_LINUX_GENERIC = 0x80000;

src/Ryujinx.Memory/MemoryProtectionException.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.InteropServices;
 
 namespace Ryujinx.Memory
 {
@@ -8,7 +9,7 @@ namespace Ryujinx.Memory
         {
         }
 
-        public MemoryProtectionException(MemoryPermission permission) : base($"Failed to set memory protection to \"{permission}\".")
+        public MemoryProtectionException(MemoryPermission permission) : base($"Failed to set memory protection to \"{permission}\": {Marshal.GetLastPInvokeErrorMessage()}")
         {
         }
 

src/Ryujinx.Memory/WindowsShared/WindowsApi.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Runtime.InteropServices;
+using System.Runtime.Versioning;
 
 namespace Ryujinx.Memory.WindowsShared
 {
+    [SupportedOSPlatform("windows")]
     static partial class WindowsApi
     {
         public static readonly IntPtr InvalidHandleValue = new IntPtr(-1);

src/Ryujinx.Memory/WindowsShared/WindowsApiException.cs

@@ -1,7 +1,9 @@
 using System;
+using System.Runtime.Versioning;
 
 namespace Ryujinx.Memory.WindowsShared
 {
+    [SupportedOSPlatform("windows")]
     class WindowsApiException : Exception
     {
         public WindowsApiException()