ssl: Work around missing remote hostname for authentication (#5988)

* ssl: Retrieve remote hostnames if the provided hostname is empty

 This avoids crashing with an AuthenticationException.

* ssl: Remove unused variable from RetrieveHostName

Directory.Packages.props

@@ -20,10 +20,10 @@
     <PackageVersion Include="jp2masa.Avalonia.Flexbox" Version="0.3.0-beta.4" />
     <PackageVersion Include="LibHac" Version="0.19.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" />
-    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.7.0" />
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" />
     <PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.2.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="2.3.2" />
+    <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.0" />
     <PackageVersion Include="MsgPack.Cli" Version="1.0.1" />
     <PackageVersion Include="NetCoreServer" Version="7.0.0" />
     <PackageVersion Include="NUnit" Version="3.13.3" />

src/Ryujinx.HLE/HOS/Services/Ssl/SslService/SslManagedSocketConnection.cs

@@ -3,6 +3,7 @@ using Ryujinx.HLE.HOS.Services.Sockets.Bsd.Impl;
 using Ryujinx.HLE.HOS.Services.Ssl.Types;
 using System;
 using System.IO;
+using System.Net;
 using System.Net.Security;
 using System.Net.Sockets;
 using System.Security.Authentication;
@@ -83,10 +84,40 @@ namespace Ryujinx.HLE.HOS.Services.Ssl.SslService
         }
 #pragma warning restore SYSLIB0039
 
+        /// <summary>
+        /// Retrieve the hostname of the current remote in case the provided hostname is null or empty.
+        /// </summary>
+        /// <param name="hostName">The current hostname</param>
+        /// <returns>Either the resolved or provided hostname</returns>
+        /// <remarks>
+        /// This is done to avoid getting an <see cref="System.Security.Authentication.AuthenticationException"/>
+        /// as the remote certificate will be rejected with <c>RemoteCertificateNameMismatch</c> due to an empty hostname.
+        /// This is not what the switch does!
+        /// It might just skip remote hostname verification if the hostname wasn't set with <see cref="ISslConnection.SetHostName"/> before.
+        /// TODO: Remove this as soon as we know how the switch deals with empty hostnames
+        /// </remarks>
+        private string RetrieveHostName(string hostName)
+        {
+            if (!string.IsNullOrEmpty(hostName))
+            {
+                return hostName;
+            }
+
+            try
+            {
+                return Dns.GetHostEntry(Socket.RemoteEndPoint.Address).HostName;
+            }
+            catch (SocketException)
+            {
+                return hostName;
+            }
+        }
+
         public ResultCode Handshake(string hostName)
         {
             StartSslOperation();
             _stream = new SslStream(new NetworkStream(((ManagedSocket)Socket).Socket, false), false, null, null);
+            hostName = RetrieveHostName(hostName);
             _stream.AuthenticateAsClient(hostName, null, TranslateSslVersion(_sslVersion), false);
             EndSslOperation();