C# SslStream throws "Verification failure during handshake"

I figured out how to make this work:

If you use a client certificate, can simply be a self-signed one you supply with the client application, then the handshake does go well. I have now also successfully got it working with a LetsEncrypt cert on the server and a self-signed one on the client.

Tested with Unity 2021.3.7f1, both in 2 editors and with the server running on Ubuntu Linux as a Unity Dedicated Server build.

1. Generate a self-signed pfx cert:

   • Create private key:

     openssl genrsa -out key.pem 2048

   • Create certifcate signing request (csr)

     openssl req -new -sha256 -key key.pem -out csr.csr

   • Create certificate

     openssl req -x509 -sha256 -days 365 -key key.pem -in csr.csr -out certificate.pem

   • Convert to .pfx file

     openssl pkcs12 -export -inkey key.pem -in certificate.pem -out certificate.pfx

   Note the password you set on the pfx file, to use in your C# code.

2. Change the code for creating the SslStream and for AuthenticateAsClient and AuthenticateAsServer and include a LocalCertificateSelectionCallback.

   Here is an example of what I got working:

   (Specifically for Unity this example also includes storing the certificates in a TextAsset (just name it like certificate.pfx.txt), using a Coroutine to start the client, and using async on the server side so it doesn't block)

   Server side

   [SerializeField] int serverPort = 4578;
   [SerializeField] TextAsset certificateFile;
   
   void RunServer()
   {
       X509Certificate2 serverCertificate = new(certificateFile.bytes, "password");
       TcpListener listener = new(IPAddress.Any, serverPort);
       listener.Start();
       listener.BeginAcceptTcpClient(new AsyncCallback(ConnectCallback), listener);
   }
   
   private void ConnectCallback(IAsyncResult result)
   {
       TcpListener listener = (TcpListener)result.AsyncState;
       TcpClient tcpClient = listener.EndAcceptTcpClient(result);
       ProcessClient(tcpClient);
   
       listener.BeginAcceptTcpClient(new AsyncCallback(ConnectCallback), listener);
   }
   
   void ProcessClient(TcpClient tcpClient)
   {
      ClientConnection client = new() { tcpClient = tcpClient };
   
       // A client has connected. Create the
       // SslStream using the client's network stream.
       client.stream = new(
           client.tcpClient.GetStream(),
           false,
           new RemoteCertificateValidationCallback(ValidateRemoteCertificate),
           new LocalCertificateSelectionCallback(CertificateSelectionCallback)
           );
       try
       {
           client.stream.AuthenticateAsServer(serverCertificate, true, false);
   
           client.stream.BeginRead(client.buffer, 0, ClientConnection.BUFFER_SIZE,
               new AsyncCallback(ReadCallback), client);
       }
       catch (AuthenticationException e)
       {
           Debug.LogException(e);
           if (e.InnerException != null)
           {
               Debug.LogError($"Inner exception: {e.InnerException.Message}");
           }
           Debug.LogWarning("Authentication failed - closing connection");
           client.stream.Close();
           tcpClient.Close();
           return;
       }
   }
   
   bool ValidateRemoteCertificate(
         object sender,
         X509Certificate certificate,
         X509Chain chain,
         SslPolicyErrors sslPolicyErrors)
   {
       // Allow any client certificate
       return true;
   }
   
   X509Certificate CertificateSelectionCallback(object sender, string targetHost, X509CertificateCollection localCertificates, X509Certificate remoteCertificate, string[] acceptableIssuers)
   {
       return localCertificates[0];
   }
   

   State object class for async:

   public class ClientConnection
   {
       internal TcpClient tcpClient = null;
       internal SslStream stream = null;
   
       // These are used for reading from the stream async
       internal const int BUFFER_SIZE = 1024;
       internal byte[] buffer = new byte[BUFFER_SIZE];
       internal StringBuilder stringBuffer = new();
   }
   

   Client side

   [SerializeField] string serverAddress;
   [SerializeField] TextAsset clientCertificate;
   TcpClient tcpClient;
   SslStream stream;
   
   void RunClient()
   {
       StartCoroutine(nameof(RunClientCoroutine));
   }
   
   IEnumerator RunClientCoroutine()
   {
       yield return new WaitForFixedUpdate();
   
       // Create a TCP/IP client socket.
       // machineName is the host running the server application.
       tcpClient = new(serverAddress, serverPort);
   
       // Create an SSL stream that will close the client's stream.
       stream = new(
           client.tcpClient.GetStream(),
           false,
           new RemoteCertificateValidationCallback(ValidateServerCertificate),
            new LocalCertificateSelectionCallback(CertificateSelectionCallback)
           );
       // The server name must match the name on the server certificate.
       try
       {
           bool authenticationPassed = true;
           X509Certificate2 cert = new(clientCertificate.bytes, "password");
           X509Certificate2Collection certs = new();
           certs.Add(cert);
           stream.AuthenticateAsClient(serverAddress, certs, SslProtocols.Tls, true);
           authenticationPassed = client.stream.IsAuthenticated;
   
           if(authenticationPassed) StartClient();
       }
       catch (AuthenticationException e)
       {
           Debug.LogException(e);
           if (e.InnerException != null)
           {
               Debug.LogError($"Inner exception: {e.InnerException.Message}");
           }
           Debug.LogWarning("Authentication failed - closing connection");
           tcpClient.Close();
       }
   }
   
   // The following method is invoked by the RemoteCertificateValidationDelegate.
   bool ValidateServerCertificate(
       object sender,
       X509Certificate certificate,
       X509Chain chain,
       SslPolicyErrors sslPolicyErrors)
   {
       if (sslPolicyErrors == SslPolicyErrors.None)
           return true;
   
   #if UNITY_EDITOR
       if (sslPolicyErrors == SslPolicyErrors.RemoteCertificateNameMismatch && (serverAddress == "localhost" || serverAddress == "127.0.0.1" || serverAddress == "::1"))
           return true;
   #endif
   
       Debug.LogError($"Certificate error: {sslPolicyErrors}");
   
       // Do not allow this client to communicate with unauthenticated servers.
       return false;
   }
   
   X509Certificate CertificateSelectionCallback(object sender, string targetHost, X509CertificateCollection localCertificates, X509Certificate remoteCertificate, string[] acceptableIssuers)
   {
       return localCertificates[0];
   }
   

For production, to convert a LetsEncrypt cert already running on a unix server from certbot apache to a pfx file for C# X509, it would be something like this:

sudo openssl pkcs12 -inkey /etc/letsencrypt/live/<domain>/privkey.pem -in /etc/letsencrypt/live/<domain>/fullchain.pem -export -out unitykey.pfx

Looks like a Unity Bug here (calling VerifyCertificate() with null and throwing an exception when null is not recognized as a valid certificate):

https://github.com/Unity-Technologies/mono/blob/06e6ccece853a5373a6545a821ca178f4ec57f20/mcs/class/System/Mono.UnityTls/UnityTlsContext.cs#L344