TcpClient only connects if server starts listening before ConnectAsync called

I have a basic TCP client and server both running locally on my machine.

If the server is already listening when I call ConnectAsync() then the client connects to the server, no problem.

If I start up the client and call ConnectAsync(), THEN start the server after to listen to clients, it doesn't establish a connection. ConnectAsync hangs for about 85 seconds before I get the following error: System.Net.Sockets.SocketException (0x80004005): Connection refused. This is despite the fact that the server has already started listening.

Not sure if this affects TcpClient but the client is running in an Xamarin project.

Here is my code:

CLIENT:

public static class DataSource
{
    private static TcpClient client;
    private static NetworkStream networkStream;

    public static bool Connected => client is null ? false : client.Connected;

    public static async Task EstablishTcpConnection(string serverIP, int port)
    {
        CloseTcpConnection();

        try
        {
            client = new TcpClient();
            await client.ConnectAsync(IPAddress.Parse(serverIP), port);
            networkStream = client.GetStream();
        }
        catch (Exception ex)
        {
            Console.WriteLine($"{nameof(EstablishTcpConnection)} Error: {ex}");
        }
    }

    public static void CloseTcpConnection()
    {
        if (networkStream != null)
        {
            networkStream.Close();
            networkStream.Dispose();
            networkStream = null;
        }

        if (client != null)
        {
            client.Close();
            client.Dispose();
            client = null;
        }
    }

    public static async Task SendTcp(string toSend)
    {
        if (client is null) return;
        if (networkStream is null) return;
        if (!Connected) return;

        if (networkStream != null && networkStream.CanWrite)
        {
            byte[] bytesToSend = ASCIIEncoding.ASCII.GetBytes(toSend);
            await networkStream.WriteAsync(bytesToSend, 0, bytesToSend.Length);
        }
    }

    public static async Task TcpListener()
    {
        while (networkStream != null && networkStream.CanRead)
        {
            if (client is null) return;
            if (networkStream is null) return;
            if (!Connected) return;

            byte[] bytesToRead = new byte[client.ReceiveBufferSize];
            int bytesRead = await networkStream.ReadAsync(bytesToRead, 0, client.ReceiveBufferSize);
            string received = Encoding.ASCII.GetString(bytesToRead, 0, bytesRead);
            Console.WriteLine($"Received: {received}");
        }
    }
}

SERVER:

internal class Program
{
    private const string serverIP = "MyServerIp";
    private const int port = myPort;

    private static void Main(string[] args)
    {
        Listener();
        Console.ReadLine();
    }

    public static async Task Listener()
    {
        //---listen at the specified IP and port no.---
        TcpListener listener = new TcpListener(IPAddress.Parse(serverIP), port);
        listener.Start();
        Console.WriteLine("Listening...");

        while (true)
        {
            //---incoming client connected---
            ReceiveClient(await listener.AcceptTcpClientAsync());
        }
    }

    public static async Task ReceiveClient(TcpClient client)
    {
        if (client is null) return;
        Console.WriteLine("Client Connected");

        //---get the incoming data through a network stream---
        NetworkStream networkStream = client.GetStream();
        byte[] buffer = new byte[client.ReceiveBufferSize];

        while (client != null && client.Connected)
        {
            //---read incoming stream---
            int bytesRead = await networkStream.ReadAsync(buffer, 0, client.ReceiveBufferSize);
            if (bytesRead == 0) break;

            //---convert the data received into a string---
            string dataReceived = Encoding.ASCII.GetString(buffer, 0, bytesRead);
            Console.WriteLine("Received : " + dataReceived);

            //---write back the text to the client---
            Console.WriteLine("Sending back : " + dataReceived);
            networkStream.Write(buffer, 0, bytesRead);
        }

        client.Close();
        client.Dispose();
        Console.WriteLine("Client Disconnected");
    }
}